leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 2.1 (#83)

Browse Source 
CUTLASS 2.1 contributes:
- BLAS-style host-side API added to CUTLASS Library
- Planar Complex GEMM kernels targeting Volta and Turing Tensor Cores
- Minor enhancements and bug fixes
This commit is contained in:
Andrew Kerr
2020-04-07 13:51:25 -07:00
committed by GitHub
parent 7c0cd26d13
commit 96dab34ad9
196 changed files with 20653 additions and 1995 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/CMakeLists.txt
+12 -3
View File
@@ -33,7 +33,7 @@ target_link_libraries(
PUBLIC
CUTLASS
cutlass_tools_util_includes
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:cublas>
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:nvidia::cublas>
gtest
)
@@ -48,6 +48,8 @@ target_link_libraries(
PUBLIC
cutlass_test_unit_infra
)
set(CUTLASS_INSTALL_TESTS ON CACHE BOOL "Install test executables")
function(cutlass_test_unit_add_executable)
@@ -65,7 +67,7 @@ function(cutlass_test_unit_add_executable)
PRIVATE
cutlass_test_unit_infra
cutlass_test_unit_infra_lib
)
)
string(REGEX REPLACE cutlass_ "" NAME_STEM ${NAME})
@@ -79,7 +81,14 @@ function(cutlass_test_unit_add_executable)
${NAME}
)
# message(STATUS "cutlass_test_unit_add_executable(${NAME} c${NAME_STEM} ${NAME_STEM})")
if (CUTLASS_INSTALL_TESTS)
install(
TARGETS ${NAME}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
endif()
endfunction()
test/unit/common/filter_architecture.cpp
+6 -21
View File
@@ -64,37 +64,22 @@ void FilterArchitecture() {
/// Maximum compute capability for which the kernels are enabled
int max_compute_capability;
/// If true, architecture is assumed to be silicon
bool silicon;
}
test_filters[] = {
{ "SM50*", 50, kMaxDevice, true},
{ "SM60*", 60, kMaxDevice, true},
{ "SM61*", 61, kMaxDevice, true},
{ "SM70*", 70, 75, true},
{ "SM75*", 75, kMaxDevice, true},
{ "SM50*", 50, kMaxDevice},
{ "SM60*", 60, kMaxDevice},
{ "SM61*", 61, kMaxDevice},
{ "SM70*", 70, 75},
{ "SM75*", 75, kMaxDevice},
{ 0, 0, false }
};
bool running_on_silicon = false;
for (int i = 0; test_filters[i].filter; ++i) {
if (deviceMajorMinor == test_filters[i].min_compute_capability) {
running_on_silicon = test_filters[i].silicon;
break;
}
}
// Set negative test filters
std::stringstream ss;
ss << "-";
for (int i = 0, j = 0; test_filters[i].filter; ++i) {
if (!running_on_silicon && deviceMajorMinor != test_filters[i].min_compute_capability) {
ss << (j++ ? ":" : "") << test_filters[i].filter;
}
else if (deviceMajorMinor < test_filters[i].min_compute_capability ||
if (deviceMajorMinor < test_filters[i].min_compute_capability ||
deviceMajorMinor > test_filters[i].max_compute_capability) {
ss << (j++ ? ":" : "") << test_filters[i].filter;
test/unit/core/functional.cu
+13 -10
View File
@@ -347,13 +347,13 @@ TEST(Functional, divides_f16x17) {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <int kN>
void Functional_multiply_add_f16xN() {
template <typename T, int kN>
void Functional_multiply_add_TxN() {
using Element = cutlass::Array<cutlass::half_t, kN>;
using Element = cutlass::Array<T, kN>;
using Operator = cutlass::multiply_add<Element>;
using Tensor = cutlass::HostTensor<cutlass::half_t, cutlass::layout::RowMajor>;
using Tensor = cutlass::HostTensor<T, cutlass::layout::RowMajor>;
Tensor D({1, kN});
Tensor A({1, kN});
@@ -361,10 +361,10 @@ void Functional_multiply_add_f16xN() {
Tensor C({1, kN});
for (int i = 0; i < kN; ++i) {
A.host_data()[i] = cutlass::half_t((i * 2 + 1) % 5);
B.host_data()[i] = cutlass::half_t((i * 4 + 8) % 7);
C.host_data()[i] = cutlass::half_t((i * 3 + 11) % 11);
D.host_data()[i] = cutlass::half_t(0);
A.host_data()[i] = T((i * 2 + 1) % 5);
B.host_data()[i] = T((i * 4 + 8) % 7);
C.host_data()[i] = T((i * 3 + 11) % 11);
D.host_data()[i] = T(0);
}
D.sync_device();
@@ -399,12 +399,15 @@ void Functional_multiply_add_f16xN() {
EXPECT_TRUE(some_d_nonzero);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Functional, multiply_add_f16x16) {
Functional_multiply_add_f16xN<16>();
Functional_multiply_add_TxN<cutlass::half_t, 16>();
}
TEST(Functional, multiply_add_f16x17) {
Functional_multiply_add_f16xN<17>();
Functional_multiply_add_TxN<cutlass::half_t, 17>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/epilogue/CMakeLists.txt
Regular → Executable
+1 -1
View File
@@ -38,4 +38,4 @@ add_custom_target(
test_unit_epilogue_thread
test_unit_epilogue_warp
test_unit_epilogue_threadblock
)
)
test/unit/epilogue/thread/CMakeLists.txt
+2 -1
View File
@@ -23,4 +23,5 @@
cutlass_test_unit_add_executable(
cutlass_test_unit_epilogue_thread
linear_combination.cu
)
linear_combination_planar_complex.cu
)
test/unit/epilogue/thread/linear_combination_planar_complex.cu
+280
View File
@@ -0,0 +1,280 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit tests for thread-level GEMM
*/
#include "../../common/cutlass_unit_test.h"
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace epilogue {
namespace thread {
using FunctorPlanarComplexF32F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
float,
4,
float,
float>;
__global__ void epilogue_thread_functor_planar_complex_f32_f32(
float *output_ptr,
float const *accum_ptr,
float const *source_ptr,
typename FunctorPlanarComplexF32F32::Params params) {
FunctorPlanarComplexF32F32 linear_combination_op(params);
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4> const *>(source_ptr);
*reinterpret_cast<cutlass::ArrayPlanarComplex<float, 4>*>(output_ptr) = linear_combination_op(accum, source);
}
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_thread_linear_combination_planar_complex, f32) {
using Element = float;
using ElementOutput = float;
int const kCount = 4;
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kCount,
Element,
Element>;
cutlass::complex<Element> alpha(Element(2), Element(1));
cutlass::complex<Element> beta(Element(1), Element(-1));
typename Functor::Params params(alpha, beta);
Functor linear_combination_op(params);
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
cutlass::ArrayPlanarComplex<Element, kCount> accum;
// Define arbitrary inputs
for (int i = 0; i < kCount; ++i) {
accum.real[i] = Element(i * 2);
accum.imag[i] = Element((i * 3 % 6) - 3);
source.real[i] = ElementOutput((i * 7 % 9) - 4);
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
}
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
// Verify each result
for (int i = 0; i < kCount; ++i) {
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace epilogue {
namespace thread {
using FunctorPlanarComplexF16F32 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
cutlass::half_t,
4,
float,
float>;
__global__ void epilogue_thread_functor_planar_complex_f16_f32(
cutlass::half_t *output_ptr,
float const *accum_ptr,
cutlass::half_t const *source_ptr,
typename FunctorPlanarComplexF16F32::Params params,
int N) {
FunctorPlanarComplexF16F32 linear_combination_op(params);
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<float , 4> const *>(accum_ptr);
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
#pragma unroll 1
for (int n = 0; n < N; ++n) {
source = linear_combination_op(accum, source);
}
*reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
}
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_thread_linear_combination_planar_complex, f16_f32) {
using Element = float;
using ElementOutput = cutlass::half_t;
int const kCount = 4;
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kCount,
Element,
Element>;
cutlass::complex<Element> alpha(Element(2), Element(1));
cutlass::complex<Element> beta(Element(1), Element(-1));
typename Functor::Params params(alpha, beta);
Functor linear_combination_op(params);
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
cutlass::ArrayPlanarComplex<Element, kCount> accum;
// Define arbitrary inputs
for (int i = 0; i < kCount; ++i) {
accum.real[i] = Element(i * 2);
accum.imag[i] = Element((i * 3 % 6) - 3);
source.real[i] = ElementOutput((i * 7 % 9) - 4);
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
}
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
// Verify each result
for (int i = 0; i < kCount; ++i) {
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace epilogue {
namespace thread {
using FunctorPlanarComplexF16F16 = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
cutlass::half_t,
4,
cutlass::half_t,
cutlass::half_t>;
__global__ void epilogue_thread_functor_planar_complex_f16_f16(
cutlass::half_t *output_ptr,
cutlass::half_t const *accum_ptr,
cutlass::half_t const *source_ptr,
typename FunctorPlanarComplexF16F16::Params params,
int N) {
FunctorPlanarComplexF16F16 linear_combination_op(params);
auto accum = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(accum_ptr);
auto source = *reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4> const *>(source_ptr);
#pragma unroll 1
for (int n = 0; n < N; ++n) {
source = linear_combination_op(accum, source);
}
*reinterpret_cast<cutlass::ArrayPlanarComplex<cutlass::half_t , 4>*>(output_ptr) = source;
}
}
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_thread_linear_combination_planar_complex, f16_f16) {
using Element = cutlass::half_t;
using ElementOutput = cutlass::half_t;
int const kCount = 8;
using Functor = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kCount,
Element,
Element>;
cutlass::complex<Element> alpha(Element(2), Element(1));
cutlass::complex<Element> beta(Element(1), Element(-1));
typename Functor::Params params(alpha, beta);
Functor linear_combination_op(params);
cutlass::ArrayPlanarComplex<ElementOutput, kCount> source;
cutlass::ArrayPlanarComplex<Element, kCount> accum;
// Define arbitrary inputs
for (int i = 0; i < kCount; ++i) {
accum.real[i] = Element(i * 2);
accum.imag[i] = Element((i * 3 % 6) - 3);
source.real[i] = ElementOutput((i * 7 % 9) - 4);
source.imag[i] = ElementOutput(((i * 5 + 2) % 9) - 4);
}
cutlass::ArrayPlanarComplex<ElementOutput, kCount> destination = linear_combination_op(accum, source);
// Verify each result
for (int i = 0; i < kCount; ++i) {
cutlass::complex<Element> expected = alpha * cutlass::complex<Element>(accum.real[i], accum.imag[i]) +
beta * cutlass::complex<Element>(Element(source.real[i]), Element(source.imag[i]));
cutlass::complex<ElementOutput> got(destination.real[i], destination.imag[i]);
EXPECT_TRUE(ElementOutput(expected.real()) == got.real());
EXPECT_TRUE(ElementOutput(expected.imag()) == got.imag());
EXPECT_TRUE(expected.real() != Element(0) || expected.imag() != Element(0));
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/epilogue/threadblock/CMakeLists.txt
Regular → Executable
+2 -1
View File
@@ -30,4 +30,5 @@ cutlass_test_unit_add_executable(
epilogue_tensor_op.cu
epilogue_volta_tensor_op.cu
epilogue_wmma_tensor_op_sm70.cu
)
epilogue_planar_complex.cu
)
test/unit/epilogue/threadblock/epilogue_planar_complex.cu
+506
View File
@@ -0,0 +1,506 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit tests for thread-level GEMM
*/
#include <fstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/aligned_buffer.h"
#include "cutlass/half.h"
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
// Tensor Op
#include "cutlass/gemm/warp/default_mma_tensor_op.h"
// Volta Tensor Op
#include "cutlass/gemm/warp/mma_tensor_op_sm70.h"
#include "cutlass/epilogue/warp/fragment_iterator_volta_tensor_op.h"
// Simt
#include "cutlass/gemm/warp/mma_simt.h"
#include "cutlass/gemm/warp/mma_simt_policy.h"
// Epilogue components
#include "cutlass/epilogue/threadblock/default_epilogue_planar_complex.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "testbed_planar_complex.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_f32_f32_tensor_op_64x64_32x32x8) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = float;
using ElementAccumulator = float;
using ElementCompute = float;
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
using Element = cutlass::half_t;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape,
InstructionShape,
Element, LayoutA,
Element, LayoutB,
ElementAccumulator, cutlass::layout::RowMajor
>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaTensorOp,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_f16_f32_tensor_op_64x64_32x32x8) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using ElementCompute = float;
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
using Element = cutlass::half_t;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape,
InstructionShape,
Element, LayoutA,
Element, LayoutB,
ElementAccumulator, cutlass::layout::RowMajor
>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaTensorOp,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_f16_f16_tensor_op_64x64_32x32x8) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using ElementCompute = cutlass::half_t;
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
using Element = cutlass::half_t;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape,
InstructionShape,
Element, LayoutA,
Element, LayoutB,
ElementAccumulator, cutlass::layout::RowMajor
>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaTensorOp,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_f32_f32_volta_tensor_op_64x64_32x32x4) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = float;
using ElementAccumulator = float;
using ElementCompute = float;
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<32, 32, 4>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 4>;
using Element = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous<cutlass::sizeof_bits<Element>::value>;
using LayoutB = cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous<cutlass::sizeof_bits<Element>::value>;
using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
cutlass::arch::Mma<
cutlass::gemm::GemmShape<16, 16, 4>,
32,
Element,
cutlass::layout::ColumnMajor,
Element,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::layout::RowMajor,
cutlass::arch::OpMultiplyAdd
>,
cutlass::MatrixShape<1, 1>
>;
using WarpMmaTensorOp = cutlass::gemm::warp::MmaVoltaTensorOp<
WarpShape,
Element,
LayoutA,
Element,
LayoutB,
ElementAccumulator,
cutlass::layout::RowMajor,
Policy
>;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaTensorOp,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm70,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_simt_f32_64x64_32x32x8) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = float;
using ElementAccumulator = float;
using ElementCompute = float;
int const kElementsPerAccess = 1;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
using Element = float;
using ElementC = ElementAccumulator;
using LayoutA = cutlass::layout::ColumnMajor;
using LayoutB = cutlass::layout::RowMajor;
using LayoutC = cutlass::layout::RowMajor;
using ElementOutput = Element;
using ElementAccumulator = Element;
using ElementCompute = Element;
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
WarpShape,
Element,
LayoutA,
Element,
LayoutB,
Element,
LayoutC,
cutlass::gemm::warp::MmaSimtPolicy<
cutlass::MatrixShape<4, 8>,
cutlass::layout::RowMajorInterleaved<2>,
cutlass::gemm::GemmShape<4, 4, 1>
>
>;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaSimt,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm50,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Epilogue_threadblock_epilogue, planar_complex_simt_f64_64x64_16x32x8) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = double;
using ElementAccumulator = double;
using ElementCompute = double;
int const kElementsPerAccess = 1;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
using Element = double;
using ElementC = ElementAccumulator;
using LayoutA = cutlass::layout::ColumnMajor;
using LayoutB = cutlass::layout::RowMajor;
using LayoutC = cutlass::layout::RowMajor;
using ElementOutput = Element;
using ElementAccumulator = Element;
using ElementCompute = Element;
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
WarpShape,
Element,
LayoutA,
Element,
LayoutB,
Element,
LayoutC,
cutlass::gemm::warp::MmaSimtPolicy<
cutlass::MatrixShape<4, 8>,
cutlass::layout::RowMajorInterleaved<2>,
cutlass::gemm::GemmShape<4, 4, 1>
>
>;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
Shape,
WarpMmaSimt,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm50,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpiloguePlanarComplexTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/epilogue/threadblock/epilogue_tensor_op.cu
+536
View File
@@ -34,6 +34,7 @@
#include "cutlass/half.h"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "cutlass/epilogue/thread/linear_combination_clamp.h"
#include "cutlass/gemm/warp/default_mma_tensor_op.h"
#include "cutlass/epilogue/threadblock/default_epilogue_tensor_op.h"
@@ -45,6 +46,541 @@
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x64_64x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x64_32x32x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_128x128_64x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<128, 128, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_128x64_64x32x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<128, 64, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 32, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x128_32x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<64, 128, 32>;
using WarpShape = cutlass::gemm::GemmShape<32, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_32x128_32x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<32, 128, 32>;
using WarpShape = cutlass::gemm::GemmShape<32, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_128x32_64x32x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<128, 32, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 32, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_256x128_64x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<256, 128, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_128x256_64x64x32) {
//
// Define the warp-level matrix multiply
//
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int;
using ElementCompute = float;
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
int const kPartitionsK = 1;
using Shape = cutlass::gemm::GemmShape<128, 256, 32>;
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
using Element = ElementOutput;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<Element>::value, 64>;
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
//
// Output operator
//
using OutputOp = cutlass::epilogue::thread::LinearCombination<
ElementOutput,
kElementsPerAccess,
ElementAccumulator,
ElementCompute
>;
//
// Define the epilogue
//
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
Shape,
WarpMmaTensorOp,
kPartitionsK,
OutputOp,
kElementsPerAccess
>::Epilogue;
//
// Instantiate epilogue
//
EpilogueTestbed<Epilogue> testbed;
bool passed = testbed.run_all();
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_64x64_64x64x16) {
//
test/unit/epilogue/threadblock/testbed_planar_complex.h
+388
View File
@@ -0,0 +1,388 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit tests for epilogues
*/
#pragma once
#include <fstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/aligned_buffer.h"
#include "cutlass/half.h"
#include "cutlass/complex.h"
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
#include "cutlass/util/host_tensor_planar_complex.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace kernel {
template <typename Epilogue>
__global__ void epilogue_planar_complex_threadblock(
typename Epilogue::OutputTileIterator::Params params_D,
typename Epilogue::OutputTileIterator::Element *ptr_D,
int64_t imaginary_stride_D,
typename Epilogue::OutputTileIterator::Params params_C,
typename Epilogue::OutputTileIterator::Element *ptr_C,
int64_t imaginary_stride_C,
typename Epilogue::OutputOp::Params params_output_op,
cutlass::MatrixCoord problem_size,
cutlass::TensorRef<
typename Epilogue::WarpMmaOperator::ElementC,
typename Epilogue::WarpMmaOperator::LayoutC> accumulator_ref,
int64_t imaginary_stride_accum,
int epilogue_count = 1) {
__shared__ typename Epilogue::SharedStorage shared_storage;
int thread_idx = threadIdx.x;
int warp_idx = threadIdx.x / 32;
int lane_idx = threadIdx.x % 32;
//
// Construct the epilogue
//
// Tile iterator writing to output tile
typename Epilogue::OutputTileIterator iterator_D_real(
params_D,
ptr_D,
problem_size,
thread_idx
);
typename Epilogue::OutputTileIterator iterator_D_imag(
params_D,
ptr_D + imaginary_stride_D,
problem_size,
thread_idx
);
// Tile iterator writing to output tile
typename Epilogue::OutputTileIterator iterator_C_real(
params_C,
ptr_C,
problem_size,
thread_idx
);
typename Epilogue::OutputTileIterator iterator_C_imag(
params_C,
ptr_C + imaginary_stride_C,
problem_size,
thread_idx
);
// Epilogue operator
Epilogue epilogue(
shared_storage,
thread_idx,
warp_idx,
lane_idx);
//
// Initialize the accumulators
//
int warp_mn = warp_idx % (Epilogue::WarpCount::kM * Epilogue::WarpCount::kN);
int warp_m = warp_mn % Epilogue::WarpCount::kM;
int warp_n = warp_mn / Epilogue::WarpCount::kM;
accumulator_ref.add_coord_offset({
warp_m * Epilogue::WarpMmaOperator::Shape::kM,
warp_n * Epilogue::WarpMmaOperator::Shape::kN});
//
// Load accumulators
//
typename Epilogue::WarpMmaOperator::IteratorC accumulator_iterator(accumulator_ref, lane_idx);
typename Epilogue::AccumulatorTile accumulators;
accumulators.clear();
accumulator_iterator.load(accumulators.real);
accumulator_iterator.load_with_pointer_offset(accumulators.imag, imaginary_stride_accum);
//
// Perform the epilogue operation
//
typename Epilogue::OutputOp output_op(params_output_op);
// Place the epilogue in a loop so assembly is clearly visible
for (int iter = 0; iter < epilogue_count; ++iter) {
epilogue(
output_op,
iterator_D_real,
iterator_D_imag,
accumulators,
iterator_C_real,
iterator_C_imag);
}
}
} // namespace kernel
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
template <
typename Epilogue_
>
class EpiloguePlanarComplexTestbed {
public:
using Epilogue = Epilogue_;
using ElementAccumulator = typename Epilogue::ElementAccumulator;
using ElementCompute = typename Epilogue::OutputOp::ElementCompute;
using ElementOutput = typename Epilogue::ElementOutput;
using OutputOpParams = typename Epilogue::OutputOp::Params;
using ComplexElementOutput = cutlass::complex<ElementOutput>;
using ComplexElementAccumulator = cutlass::complex<ElementAccumulator>;
using ComplexElementCompute = cutlass::complex<ElementCompute>;
public:
//
// Data members
//
cutlass::MatrixCoord quantized_size;
cutlass::HostTensorPlanarComplex<ElementAccumulator, cutlass::layout::RowMajor> accumulator_tensor;
cutlass::HostTensorPlanarComplex<ElementOutput, cutlass::layout::RowMajor> source_tensor;
cutlass::HostTensorPlanarComplex<ElementOutput, cutlass::layout::RowMajor> output_tensor;
public:
//
// Methods
//
EpiloguePlanarComplexTestbed():
quantized_size(Epilogue::Shape::kM, Epilogue::Shape::kN),
accumulator_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
source_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
output_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}) {
//
// Initialize problem space
//
#if 1
uint64_t seed = 2019;
cutlass::reference::host::TensorFillRandomUniform(
accumulator_tensor.host_view(),
seed,
20,
-20,
0);
cutlass::reference::host::TensorFillRandomUniform(
source_tensor.host_view(),
seed + 2018,
20,
-20,
0);
#else
cutlass::reference::host::BlockFillSequential(accumulator_tensor.host_data(), accumulator_tensor.capacity());
#endif
}
bool run_all() {
cutlass::complex<float> alpha_values[3];
alpha_values[0] = cutlass::complex<float>(1, 0);
alpha_values[1] = cutlass::complex<float>(0, 0);
alpha_values[2] = cutlass::complex<float>(2.25f, -0.5f);
cutlass::complex<float> beta_values[3];
beta_values[0] = cutlass::complex<float>(0, 0);
beta_values[1] = cutlass::complex<float>(1, 0);
beta_values[2] = cutlass::complex<float>(0.5f, -2.25f);
// Test runtime explodes if we tried to test every case exhaustively. This tests the full
// output tile and several smaller sizes to stress predication.
for (int m_idx = 0; m_idx < 3; ++m_idx) {
for (int n_idx = 0; n_idx < 3; ++n_idx) {
cutlass::MatrixCoord problem_size(
quantized_size.row() - m_idx * 3,
quantized_size.column() - n_idx * Epilogue::kElementsPerAccess
);
for (auto const &alpha : alpha_values) {
for (auto const &beta : beta_values) {
bool passed = run(problem_size, {alpha, beta});
if (!passed) {
return false;
}
}
}
}
}
return true;
}
/// Runs the test
bool run(
cutlass::MatrixCoord problem_size,
OutputOpParams output_params) {
//
// Initialize problem space
//
ComplexElementOutput default_output = ComplexElementOutput(ElementOutput(-127), ElementOutput(-101));
cutlass::reference::host::TensorFill(output_tensor.host_view(), default_output);
accumulator_tensor.sync_device();
output_tensor.sync_device();
source_tensor.sync_device();
//
// Initialize epilogue parameters
//
typename Epilogue::OutputTileIterator::Params params_D(output_tensor.layout());
typename Epilogue::OutputTileIterator::Params params_C(source_tensor.layout());
//
// Launch kernel
//
dim3 grid(1, 1);
dim3 block(Epilogue::WarpCount::kCount * 32, 1);
test::kernel::epilogue_planar_complex_threadblock<Epilogue><<< grid, block >>>(
params_D,
output_tensor.device_data(),
output_tensor.imaginary_stride(),
params_C,
source_tensor.device_data(),
source_tensor.imaginary_stride(),
output_params,
problem_size,
accumulator_tensor.device_view_real(),
accumulator_tensor.imaginary_stride()
);
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
std::cerr << "Kernel error: " << cudaGetErrorString(result) << std::endl;
return false;
}
//
// Verify results
//
output_tensor.sync_host();
int errors = 0;
int const kMaxErrors = 5;
for (int r = 0; errors < kMaxErrors && r < quantized_size.row(); ++r) {
for (int c = 0; errors < kMaxErrors && c < quantized_size.column(); ++c) {
cutlass::MatrixCoord coord{r, c};
ComplexElementOutput got = output_tensor.at(coord);
ComplexElementOutput expected = default_output;
if (coord.row() < problem_size.row() && coord.column() < problem_size.column()) {
ComplexElementOutput src = source_tensor.at(coord);
ComplexElementCompute tmp =
output_params.alpha * ComplexElementCompute(accumulator_tensor.at(coord)) +
output_params.beta * ComplexElementCompute(src.real(), src.imag());
expected = ComplexElementOutput(ElementOutput(tmp.real()), ElementOutput(tmp.imag()));
}
if (expected != got) {
using OutputIO = cutlass::ScalarIO<ComplexElementOutput>;
EXPECT_TRUE(false)
<< "-------\n"
<< "Error - output element (" << coord << ") - expected: "
<< OutputIO(expected)
<< ", got: " << OutputIO(got) << std::endl;
++errors;
}
}
}
//
// Report results on error
//
if (errors) {
std::cout << "Incorrect result for problem("
<< problem_size.row() << ", "
<< problem_size.column() << ") for alpha: " << output_params.alpha << ", beta: " << output_params.beta << std::endl;
std::stringstream ss;
ss
<< "output_tensor_op_" << Epilogue::Shape::kM << "x" << Epilogue::Shape::kN << "_"
<< Epilogue::WarpTileIterator::WarpShape::kM << "x"
<< Epilogue::WarpTileIterator::WarpShape::kN
<< "_slice_" << Epilogue::WarpCount::kK << ".csv";
std::ofstream output_file(ss.str());
output_file << output_tensor.host_view();
std::cout << "Wrote workspace to '" << ss.str() << "'" << std::endl;
}
return !errors;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/CMakeLists.txt
+6 -3
View File
@@ -23,9 +23,13 @@
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device
BATCH_SOURCES ON
BATCH_SIZE 4
gemm_f16t_f16n_f16t_tensor_op_f16_sm75.cu
gemm_f16n_f16t_f16t_tensor_op_f16_sm75.cu
gemm_f16n_f16t_f16t_tensor_op_f16_sm75_slicedk.cu
gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
gemm_f16n_f16n_f16t_tensor_op_f32_sm75.cu
@@ -90,6 +94,7 @@ cutlass_test_unit_add_executable(
simt_zgemm_tn_sm50.cu
simt_zgemm_tt_sm50.cu
gemm_splitk_serial_tensor_op_sm75.cu
gemm_splitk_tensor_op_sm75.cu
gemm_splitk_tensor_op_sm70.cu
gemm_splitk_simt_sm50.cu
@@ -145,5 +150,3 @@ cutlass_test_unit_add_executable(
gemm_f16t_f16n_f32t_singlestage_wmma_tensor_op_f32_sm70.cu
)
test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_sm75_slicedk.cu → test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
View File
test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_sm75_slicedk.cu → test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
View File
test/unit/gemm/device/gemm_planar_complex_f16_f16_f32_tensor_op_sm70.cu
+131
View File
@@ -0,0 +1,131 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-level GEMM API for Planar Complex.
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/kernel/default_gemm_planar_complex_universal.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "testbed_planar_complex.h"
#if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
using gemm_planar_complex_s884_tn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
cutlass::half_t,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm70,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
float,
4,
float,
float
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2,
cutlass::arch::OpMultiplyAdd
>::GemmKernel;
struct gemm_planar_complex_s884_tn : gemm_planar_complex_s884_tn_base {
};
////////////////////////////////////////////////////////////////////////////////
TEST(SM70_Device_GemmPlanarComplex_f16t_f16n_f32n_tensor_op_f32_884, 64x64x32_32x32x32) {
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_tn>;
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
using gemm_planar_complex_s884_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
8,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm70,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
float,
4,
float,
float
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2,
cutlass::arch::OpMultiplyAdd
>::GemmKernel;
struct gemm_planar_complex_s884_nt : gemm_planar_complex_s884_nt_base {
};
////////////////////////////////////////////////////////////////////////////////
TEST(SM70_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_884, 64x64x32_32x32x32) {
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_nt>;
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
test/unit/gemm/device/gemm_s4t_s4n_s32n_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x256x128_64x64x128) {
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 256x128x128_64x64x128) {
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x128x128_64x64x128) {
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 64x128x128_32x64x128) {
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x64x128_64x32x128) {
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 64x64x128_32x32x128) {
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32n_wmma_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -66,7 +66,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x256x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -98,7 +98,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 256x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -162,7 +162,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 64x128x128_32x64x128_8x8x
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -194,7 +194,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x64x128_64x32x128_8x8x
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -226,7 +226,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 64x64x128_32x32x128_8x8x3
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32t_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x256x128_64x64x128) {
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x128x128_64x64x128) {
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x128_64x64x128) {
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x128x128_32x64x128) {
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x128_64x32x128) {
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x128_32x32x128) {
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32t_wmma_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -66,7 +66,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x256x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -98,7 +98,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 256x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -162,7 +162,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 64x128x128_32x64x128_8x8x
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -194,7 +194,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x64x128_64x32x128_8x8x
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -226,7 +226,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 64x64x128_32x32x128_8x8x3
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s4n_tensor_op_s32_sm75.cu
+243
View File
@@ -0,0 +1,243 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x256x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 256x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 64x128x128_32x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x64x128_64x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 64x64x128_32x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s4t_s4n_s4t_tensor_op_s32_sm75.cu
+243
View File
@@ -0,0 +1,243 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x256x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 256x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 64x128x128_32x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x64x128_64x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 64x64x128_32x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8n_s8t_s8n_tensor_op_s32_sm75.cu
+14 -28
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 32x64x64_16x32x64) {
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -101,11 +99,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -137,11 +133,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -173,11 +167,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -209,11 +201,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -245,11 +235,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -281,11 +269,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_s8t_s8n_s32n_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32n_wmma_tensor_op_s32_sm72.cu
+3 -3
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -96,7 +96,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32t_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32t_wmma_tensor_op_s32_sm72.cu
+4 -4
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -131,7 +131,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -165,7 +165,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s8n_tensor_op_s32_sm75.cu
+75 -13
View File
@@ -57,9 +57,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x256x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -77,11 +76,10 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 256x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
@@ -96,9 +94,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -116,15 +113,80 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 64x128x64_32x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x64x64_64x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t,
cutlass::layout::RowMajor,
int8_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 64x64x64_32x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t,
cutlass::layout::RowMajor,
int8_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8t_s8n_s8n_wmma_tensor_op_s32_sm72.cu
+8 -16
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16x
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -96,11 +94,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -130,11 +126,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -164,11 +158,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_s8t_s8n_s8t_tensor_op_s32_sm75.cu
+50 -12
View File
@@ -57,9 +57,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x256x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -76,9 +75,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 256x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -95,9 +93,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -115,14 +112,55 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 64x128x64_32x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x64x64_64x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t, cutlass::layout::RowMajor, int8_t, cutlass::layout::ColumnMajor,
ElementOutput, cutlass::layout::RowMajor, ElementAccumulator,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
test::gemm::device::Testbed<Gemm> testbed;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 64x64x64_32x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t, cutlass::layout::RowMajor, int8_t, cutlass::layout::ColumnMajor,
ElementOutput, cutlass::layout::RowMajor, ElementAccumulator,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
test::gemm::device::Testbed<Gemm> testbed;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8t_s8n_s8t_wmma_tensor_op_s32_sm72.cu
+8 -16
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16x
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -97,11 +95,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -131,11 +127,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -165,11 +159,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_splitk_serial_tensor_op_sm75.cu
+107
View File
@@ -0,0 +1,107 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_GemmSplitKSerial_f16n_f16n_f16t_tensor_op_f32, 128x256x32_64x64x32) {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
static const int kStages = 2;
static const int kAlignmentA = cutlass::gemm::device::DefaultGemmConfiguration<
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
ElementA,
ElementB,
ElementOutput,
ElementAccumulator>::kAlignmentA;
static const int kAlignmentB = cutlass::gemm::device::DefaultGemmConfiguration<
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
ElementA,
ElementB,
ElementOutput,
ElementAccumulator>::kAlignmentB;
static const bool kSplitKSerial = true;
using Gemm = cutlass::gemm::device::Gemm<
ElementA,
cutlass::layout::ColumnMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 8>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
kStages,
kAlignmentA,
kAlignmentB,
kSplitKSerial
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_splitk_tensor_op_sm70.cu
+1
View File
@@ -42,6 +42,7 @@
#include "testbed_splitk.h"
// These tests cannot run unless CUDA 10.1 Toolkit or later is used.
#if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/gemm_splitk_tensor_op_sm75.cu
+1
View File
@@ -42,6 +42,7 @@
#include "testbed_splitk.h"
// These tests cannot run unless CUDA 10.2 Toolkit or later is used.
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed.h
+16 -26
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
@@ -41,20 +43,7 @@
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/host/gemm.h"
inline char const *to_string(cutlass::Status status) {
switch (status) {
case cutlass::Status::kSuccess: return "kSuccess";
case cutlass::Status::kErrorMisalignedOperand: return "kErrorMisalignedOperand";
case cutlass::Status::kErrorInvalidLayout: return "kErrorInvalidLayout";
case cutlass::Status::kErrorInvalidProblem: return "kErrorInvalidProblem";
case cutlass::Status::kErrorNotSupported: return "kErrorNotSupported";
case cutlass::Status::kErrorWorkspaceNull: return "kErrorWorkspaceNull";
case cutlass::Status::kErrorInternal: return "kErrorInternal";
case cutlass::Status::kInvalid: return "kInvalid";
}
return "invalid";
}
#include "testbed_utils.h"
namespace test {
namespace gemm {
@@ -185,9 +174,12 @@ struct Testbed {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
if (tensor_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
if (reference_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
@@ -341,18 +333,12 @@ bool TestAllGemm() {
(cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::RowMajor>::value ||
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::ColumnMajor>::value) ? 4 : kAlignment;
int problem_size_m[] = {
kAlignmentM, 512 - 3*kAlignmentM
};
int problem_size_m[] = {kAlignmentM, 512 - 3 * kAlignmentM};
int problem_size_n[] = {
kAlignmentN, 512 - 2*kAlignmentN
};
int problem_size_n[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int problem_size_k[] = {
kAlignmentK, Gemm::ThreadblockShape::kK * Gemm::kStages - kAlignmentK
};
kAlignmentK, Gemm::ThreadblockShape::kK * (Gemm::kStages + 1) - kAlignmentK};
int split_k_slices[] = {
1, 2, 3
@@ -379,6 +365,10 @@ bool TestAllGemm() {
continue;
}
if (split_k > 1 && k / Gemm::ThreadblockShape::kK < split_k) {
continue;
}
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
test/unit/gemm/device/testbed_complex.h
+3
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
@@ -90,6 +92,7 @@ struct TestbedComplex : public Testbed<Gemm> {
this->tensor_B.host_ref(),
Gemm::kTransformB,
beta,
this->tensor_C.host_ref(),
this->reference_D.host_ref(),
ElementAccumulator(0)
);
test/unit/gemm/device/testbed_planar_complex.h
+283
View File
@@ -0,0 +1,283 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/gemm_planar_complex.h"
#include "cutlass/util/host_tensor_planar_complex.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
class TestbedPlanarComplex {
public:
using ElementA = typename Gemm::ElementA;
using LayoutA = typename Gemm::LayoutA;
using ElementB = typename Gemm::ElementB;
using LayoutB = typename Gemm::LayoutB;
using ElementC = typename Gemm::ElementC;
using LayoutC = typename Gemm::LayoutC;
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
using ElementAccumulator = typename Gemm::ElementAccumulator;
//
// Data members
//
cutlass::gemm::GemmCoord problem_size;
cutlass::HostTensorPlanarComplex<ElementA, LayoutA> tensor_A;
cutlass::HostTensorPlanarComplex<ElementB, LayoutB> tensor_B;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_C;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_D;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_D_ref;
//
// Methods
//
TestbedPlanarComplex(cutlass::gemm::GemmCoord const & problem_size): problem_size(problem_size) {
tensor_A.reset({problem_size.m(), problem_size.k()});
tensor_B.reset({problem_size.k(), problem_size.n()});
tensor_C.reset({problem_size.m(), problem_size.n()});
tensor_D.reset({problem_size.m(), problem_size.n()});
tensor_D_ref.reset({problem_size.m(), problem_size.n()}, false);
}
void initialize() {
uint64_t seed = 1073;
int scope_max = 8;
int scope_min = -8;
cutlass::reference::host::TensorFillRandomUniform(
tensor_A.host_view(), seed, scope_max, scope_min, 0);
cutlass::reference::host::TensorFillRandomUniform(
tensor_B.host_view(), seed * 2019, scope_max, scope_min, 0);
cutlass::reference::host::TensorFillRandomUniform(
tensor_C.host_view(), seed * 2020, scope_max, scope_min, 0);
cutlass::reference::host::TensorFill(tensor_D.host_view());
cutlass::reference::host::TensorFill(tensor_D_ref.host_view());
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D.sync_device();
}
bool run(
cutlass::complex<ElementCompute> alpha = {1, 0},
cutlass::complex<ElementCompute> beta = {0, 0}) {
initialize();
int batch_count = 1;
ElementA *ptr_A = tensor_A.device_data();
ElementB *ptr_B = tensor_B.device_data();
ElementC *ptr_C = tensor_C.device_data();
ElementC *ptr_D = tensor_D.device_data();
int lda = tensor_A.layout().stride(0);
int ldb = tensor_B.layout().stride(0);
int ldc = tensor_C.layout().stride(0);
int ldd = tensor_D.layout().stride(0);
int64_t imag_stride_A = tensor_A.imaginary_stride();
int64_t imag_stride_B = tensor_B.imaginary_stride();
int64_t imag_stride_C = tensor_C.imaginary_stride();
int64_t imag_stride_D = tensor_D.imaginary_stride();
//
// Launch device kernel
//
Gemm gemm_op;
typename Gemm::Arguments args{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
batch_count,
{alpha, beta},
ptr_A,
ptr_A + imag_stride_A,
ptr_B,
ptr_B + imag_stride_B,
ptr_C,
ptr_C + imag_stride_C,
ptr_D,
ptr_D + imag_stride_D,
lda,
lda,
ldb,
ldb,
ldc,
ldc,
ldd,
ldd
};
cutlass::Status status = gemm_op(args);
EXPECT_EQ(status, cutlass::Status::kSuccess);
cudaError_t error = cudaDeviceSynchronize();
tensor_D.sync_host();
//
// Compute reference
//
cutlass::reference::host::GemmPlanarComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementAccumulator
>(
problem_size,
alpha,
tensor_A.host_ref(),
Gemm::kTransformA,
tensor_B.host_ref(),
Gemm::kTransformB,
beta,
tensor_C.host_ref(),
tensor_D_ref.host_ref()
);
bool passed = cutlass::reference::host::TensorEquals(
tensor_D.host_view(),
tensor_D_ref.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
std::ofstream output("gemm_planar_complex.txt");
output
<< "A:\n" << tensor_A.host_view() << "\n"
<< "B:\n" << tensor_B.host_view() << "\n"
<< "C:\n" << tensor_C.host_view() << "\n"
<< "Reference:\n"
<< tensor_D_ref.host_view() << "\n"
<< "Computed:\n"
<< tensor_D.host_view() << "\n";
}
return passed;
}
};
template <typename Gemm>
bool TestOneGemmPlanarComplex(cutlass::gemm::GemmCoord problem_size) {
TestbedPlanarComplex<Gemm> testbed(problem_size);
return testbed.run();
}
template <typename Gemm>
bool TestAllGemmPlanarComplex() {
int M[] = {
16, 264,
};
int N[] = {
16, 248,
};
int K[] = {
8, 96,
};
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
cutlass::complex<ElementCompute> alpha_values[] = {
{ElementCompute(1.25), ElementCompute(-0.5)}
};
cutlass::complex<ElementCompute> beta_values[] = {
{ElementCompute(-2.25), ElementCompute(1.5)}
};
for (int m : M) {
for (int n : N) {
for (int k : K) {
test::gemm::device::TestbedPlanarComplex<Gemm> testbed({m, n, k});
for (auto const &alpha : alpha_values) {
for (auto const &beta : beta_values) {
bool passed = testbed.run(alpha, beta);
if (!passed) {
return false;
}
}
}
}
}
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace gemm
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed_splitk.h
+2
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
test/unit/gemm/device/testbed_universal.h
+480
View File
@@ -0,0 +1,480 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/gemm_complex.h"
#include "testbed_utils.h"
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
struct TestbedUniversal {
using ElementAccumulator = typename Gemm::ElementAccumulator;
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
/// Initialization
cutlass::Distribution::Kind init_A;
cutlass::Distribution::Kind init_B;
cutlass::Distribution::Kind init_C;
uint64_t seed;
cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A;
cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_C;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_D;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> reference_D;
//
// Methods
//
TestbedUniversal(
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint64_t seed_ = 2080
):
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
/// Helper to initialize a tensor view
template <typename Element, typename Layout>
bool initialize_tensor(
cutlass::TensorView<Element, Layout> view,
cutlass::Distribution::Kind dist_kind,
uint64_t seed) {
if (dist_kind == cutlass::Distribution::Uniform) {
double scope_max, scope_min;
int bits_input = cutlass::sizeof_bits<Element>::value;
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
} else if (bits_output == 16) {
scope_max = 5;
scope_min = -5;
} else {
scope_max = 8;
scope_min = -8;
}
cutlass::reference::host::TensorFillRandomUniform(
view, seed, scope_max, scope_min, 0);
}
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
}
else if (dist_kind == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(
view.data(), view.capacity());
}
else {
// TODO: Implement the rest
EXPECT_TRUE(false) << "Not implemented";
return false;
}
return true;
}
/// Initializes data structures
void initialize(cutlass::gemm::GemmCoord problem_size) {
//
// Allocate the GEMM workspace
//
tensor_A.resize(problem_size.mk());
tensor_B.resize(problem_size.kn());
tensor_C.resize(problem_size.mn());
tensor_D.resize(problem_size.mn());
reference_D.resize(problem_size.mn(), false);
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
// It is possible to randomly initialize to all zeros, so override this with non-zeros
// in the upper left corner of each operand.
tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D.sync_device();
}
/// Compares computed reference with device reference and outputs to a file if incorrect
bool compare_reference(
cutlass::gemm::GemmCoord problem_size,
ElementCompute alpha,
ElementCompute beta) {
tensor_D.sync_host();
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
EXPECT_TRUE(passed) << " mismatched reference";
if (!passed) {
/*
std::stringstream fname;
fname << "error_Gemm_device_"
<< problem_size.m() << "x"
<< problem_size.n() << "x"
<< problem_size.k() << "_"
<< Gemm::ThreadblockShape::kM << "x"
<< Gemm::ThreadblockShape::kN << "x"
<< Gemm::ThreadblockShape::kK << "_"
<< Gemm::WarpShape::kM << "x"
<< Gemm::WarpShape::kN << "x"
<< Gemm::WarpShape::kK << ".txt";
std::ofstream file(fname.str());
*/
std::ofstream file("testbed_universal_errors.txt");
file
<< "problem: " << problem_size
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
file
<< "A =\n" << tensor_A.host_view()
<< "\nB =\n" << tensor_B.host_view()
<< "\nC =\n" << tensor_C.host_view()
<< "\n\nReference =\n" << reference_D.host_view()
<< "\nComputed =\n" << tensor_D.host_view();
}
return passed;
}
/// Verifies the result is a GEMM
bool verify(
cutlass::gemm::GemmCoord problem_size,
ElementCompute alpha,
ElementCompute beta) {
//
// Verify
//
cutlass::reference::host::GemmComplex<
typename Gemm::ElementA, typename Gemm::LayoutA,
typename Gemm::ElementB, typename Gemm::LayoutB,
typename Gemm::ElementC, typename Gemm::LayoutC,
ElementCompute, ElementAccumulator
>(
problem_size,
alpha,
tensor_A.host_ref(),
Gemm::kTransformA,
tensor_B.host_ref(),
Gemm::kTransformB,
beta,
tensor_C.host_ref(),
reference_D.host_ref(),
ElementAccumulator(0)
);
return compare_reference(problem_size, alpha, beta);
}
/// Executes one test
bool run(
cutlass::gemm::GemmUniversalMode mode,
cutlass::gemm::GemmCoord problem_size,
int batch_count = 1,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(0)) {
this->initialize(problem_size);
//
// Initialize the GEMM operator
//
typename Gemm::Arguments arguments{
mode,
problem_size,
batch_count,
{alpha, beta},
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data(),
tensor_D.device_data(),
problem_size.m() * problem_size.k(),
problem_size.n() * problem_size.k(),
problem_size.m() * problem_size.n(),
problem_size.m() * problem_size.n(),
tensor_A.layout().stride(0),
tensor_B.layout().stride(0),
tensor_C.layout().stride(0),
tensor_D.layout().stride(0)
};
Gemm gemm_op;
size_t workspace_size = Gemm::get_workspace_size(arguments);
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
//
// Run the GEMM
//
status = gemm_op();
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
//
// Verify
//
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
bool TestGemmUniversal(
cutlass::gemm::GemmCoord const & problem_size,
cutlass::gemm::GemmUniversalMode mode,
int batch_count,
double alpha = 1.0,
double beta = 2.0) {
bool passed = true;
TestbedUniversal<Gemm> testbed;
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
passed = testbed.run(
mode,
problem_size,
batch_count,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
return passed;
}
template <typename Gemm>
bool TestAllGemmUniversal() {
bool passed = true;
int const kMinimumOperandElementSize =
std::min(
int(cutlass::sizeof_bits<typename Gemm::ElementA>::value),
int(cutlass::sizeof_bits<typename Gemm::ElementB>::value));
int const kAlignment = cutlass::platform::is_same<
typename Gemm::OperatorClass,
cutlass::arch::OpClassSimt>::value ? 1 : 128 / kMinimumOperandElementSize;
// int8_t gemm alignment constraints
int const kAlignmentM = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementA, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::ColumnMajor>::value ? 4 : kAlignment;
int const kAlignmentN = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementB, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::RowMajor>::value ? 4 : kAlignment;
int const kAlignmentK = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementA, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::ElementB, int8_t>::value &&
(cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::RowMajor>::value ||
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::ColumnMajor>::value) ? 4 : kAlignment;
cutlass::gemm::GemmUniversalMode modes[] = {
cutlass::gemm::GemmUniversalMode::kGemm,
};
int problem_size_m[] = {
kAlignmentM, 512 - 3*kAlignmentM
};
int problem_size_n[] = {
kAlignmentN, 512 - 2*kAlignmentN
};
int problem_size_k[] = {
kAlignmentK,
Gemm::ThreadblockShape::kK * Gemm::kStages - kAlignmentK,
Gemm::ThreadblockShape::kK * Gemm::kStages * 3 - kAlignmentK
};
int batch_counts[] = { // may be interpretted as batch count or split-K slices
1, 2, 3, 5, 7
};
double problem_alpha[] = {
1
};
double problem_beta[] = {
2.0
};
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
for (cutlass::gemm::GemmUniversalMode mode : modes) {
for (int m : problem_size_m) {
for (int n : problem_size_n) {
for (int k : problem_size_k) {
for (int batch_count : batch_counts) {
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
if (mode == cutlass::gemm::GemmUniversalMode::kGemm ||
mode == cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel) {
// skip very small K problems
if (k / batch_count < 2 * Gemm::ThreadblockShape::kK) {
continue;
}
}
cutlass::gemm::GemmCoord problem_size(m, n, k);
TestbedUniversal<Gemm> testbed;
passed = testbed.run(
mode,
problem_size,
batch_count,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
if (!passed) {
return false;
}
}
}
}
}
}
}
}
/*
// large problem with high coverage
for (int split_k_slices = 1; split_k_slices <= 3; ++split_k_slices) {
TestbedUniversal<Gemm> testbed;
cutlass::gemm::GemmCoord problem_size(72, 56, 8192);
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
split_k_slices,
cutlass::from_real<ElementCompute>(1.0),
cutlass::from_real<ElementCompute>(2.0)
);
if (!passed) {
break;
}
}
*/
return passed;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace gemm
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed_utils.h
+46
View File
@@ -0,0 +1,46 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include "cutlass/cutlass.h"
inline char const *to_string(cutlass::Status status) {
switch (status) {
case cutlass::Status::kSuccess: return "kSuccess";
case cutlass::Status::kErrorMisalignedOperand: return "kErrorMisalignedOperand";
case cutlass::Status::kErrorInvalidLayout: return "kErrorInvalidLayout";
case cutlass::Status::kErrorInvalidProblem: return "kErrorInvalidProblem";
case cutlass::Status::kErrorNotSupported: return "kErrorNotSupported";
case cutlass::Status::kErrorWorkspaceNull: return "kErrorWorkspaceNull";
case cutlass::Status::kErrorInternal: return "kErrorInternal";
case cutlass::Status::kInvalid: return "kInvalid";
}
return "invalid";
}
test/unit/gemm/threadblock/mma_planar_complex_testbed.h
+345
View File
@@ -0,0 +1,345 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
*modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice,
*this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
*notice, this list of conditions and the following disclaimer in the
*documentation and/or other materials provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its
*contributors may be used to endorse or promote products derived from this
*software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
*AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
*IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
*DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY DIRECT,
*INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
*DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
*OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TOR (INCLUDING
*NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
*EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit testbed for kernel-level GEMM
*/
#pragma once
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/platform/platform.h"
#include "cutlass/aligned_buffer.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/layout/vector.h"
#include "cutlass/numeric_types.h"
#include "cutlass/core_io.h"
#include "cutlass/util/host_tensor_planar_complex.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/gemm_planar_complex.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_fill.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace threadblock {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Mma>
__global__ void kernel_mma_planar_complex(
cutlass::gemm::GemmCoord problem_size,
typename Mma::IteratorA::Params params_A,
typename Mma::IteratorA::Element *ptr_A,
int64_t imaginary_stride_A,
typename Mma::IteratorB::Params params_B,
typename Mma::IteratorB::Element *ptr_B,
int64_t imaginary_stride_B,
typename Mma::ElementC *ptr_C, int ldc, int64_t imaginary_stride_C) {
// Shared storage needed by threadblock-scoped matrix multiply-accumulate
__shared__ typename Mma::SharedStorage shared_storage;
// Compute threadblock location
cutlass::gemm::GemmCoord tb_tile_offset = {int(blockIdx.x), int(blockIdx.y),
0};
cutlass::MatrixCoord tb_offset_A{tb_tile_offset.m() * Mma::Shape::kM,
tb_tile_offset.k()};
cutlass::MatrixCoord tb_offset_B{tb_tile_offset.k(),
tb_tile_offset.n() * Mma::Shape::kN};
// Compute position within threadblock
int tb_thread_id = threadIdx.y * blockDim.x + threadIdx.x;
// Construct iterators to A operand
typename Mma::IteratorA iterator_A_real(params_A, ptr_A,
{problem_size.m(), problem_size.k()},
tb_thread_id, tb_offset_A);
typename Mma::IteratorA iterator_A_imag(params_A, ptr_A + imaginary_stride_A,
{problem_size.m(), problem_size.k()},
tb_thread_id, tb_offset_A);
// Construct iterators to B operand
typename Mma::IteratorB iterator_B_real(params_B, ptr_B,
{problem_size.k(), problem_size.n()},
tb_thread_id, tb_offset_B);
typename Mma::IteratorB iterator_B_imag(params_B, ptr_B + imaginary_stride_B,
{problem_size.k(), problem_size.n()},
tb_thread_id, tb_offset_B);
int warp_id = threadIdx.y;
int lane_id = threadIdx.x;
// Construct thread-scoped matrix multiply
Mma mma(shared_storage, tb_thread_id, warp_id, threadIdx.x);
typename Mma::FragmentC accum;
accum.clear();
int gemm_k_iterations = (problem_size.k() + Mma::Shape::kK - 1) / Mma::Shape::kK;
// Compute threadblock-scoped matrix multiply-add
mma(gemm_k_iterations, accum, iterator_A_real, iterator_A_imag, iterator_B_real, iterator_B_imag, accum);
// Output results
typename Mma::Operator::IteratorC iterator_C({ptr_C, ldc}, lane_id);
iterator_C.add_tile_offset(
{(tb_tile_offset.m() * Mma::WarpCount::kM) +
(warp_id % Mma::WarpCount::kM),
(tb_tile_offset.n() * Mma::WarpCount::kN) +
(warp_id / Mma::WarpCount::kM)});
iterator_C.store(accum.real);
iterator_C.store_with_pointer_offset(accum.imag, imaginary_stride_C);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Threadblock-level matrix multiply-accumulate
typename Mma_>
struct TestbedPlanarComplex {
using Mma = Mma_;
using ThreadblockShape = typename Mma::Shape;
using IteratorA = typename Mma::IteratorA;
using ElementA = typename Mma::IteratorA::Element;
using LayoutA = typename Mma::IteratorA::Layout;
using IteratorB = typename Mma::IteratorB;
using ElementB = typename Mma::IteratorB::Element;
using LayoutB = typename Mma::IteratorB::Layout;
using ElementC = typename Mma::ElementC;
using ElementAccumulator = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
using ThreadMapA = typename Mma::IteratorA::ThreadMap;
using ThreadMapB = typename Mma::IteratorB::ThreadMap;
using AccessTypeA = cutlass::Array<ElementA, ThreadMapA::kElementsPerAccess>;
using AccessTypeB = cutlass::Array<ElementB, ThreadMapB::kElementsPerAccess>;
static int const Stages = Mma::kStages;
static cutlass::arch::CacheOperation::Kind const CacheOpA =
Mma::kCacheOpA;
static cutlass::arch::CacheOperation::Kind const CacheOpB =
Mma::kCacheOpB;
//
// Data members
//
cutlass::HostTensorPlanarComplex<ElementA, LayoutA> matrix_A;
cutlass::HostTensorPlanarComplex<ElementB, LayoutB> matrix_B;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> matrix_C_computed;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> matrix_C_reference;
cutlass::gemm::GemmCoord problem_size;
//
// Methods
//
/// Allocates workspace in device memory
TestbedPlanarComplex(int m, int n, int k)
: problem_size(m, n, k) {
matrix_A.reset(cutlass::make_Coord(m, k));
matrix_B.reset(cutlass::make_Coord(k, n));
matrix_C_computed.reset(cutlass::make_Coord(m, n));
matrix_C_reference.reset(cutlass::make_Coord(m, n), false);
}
/// Runs the test
bool run(
dim3 grid, dim3 block,
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementA>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementA>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
matrix_A.host_view(), seed, scope_max, scope_min, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
for (int i = 0; i < matrix_A.capacity() * 2; ++i) {
matrix_A.host_data()[i] = cutlass::half_t(float(i % 5) - 2);
}
/*
cutlass::reference::host::BlockFillSequential(matrix_A.host_data(),
matrix_A.capacity() * 2);
*/
} else if (init_A == cutlass::Distribution::Identity) {
//cutlass::reference::host::TensorFillIdentity(matrix_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementB>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementB>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
matrix_B.host_view(), seed + 16, scope_max, scope_min, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(matrix_B.host_data(),
matrix_B.capacity() * 2);
for (int i = 0; i < matrix_B.capacity() * 2; ++i) {
matrix_B.host_data()[i] = cutlass::half_t(float((i + 3) % 5) - 2);
}
} else if (init_B == cutlass::Distribution::Identity) {
//cutlass::reference::host::TensorFillIdentity(matrix_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
matrix_A.sync_device();
matrix_B.sync_device();
matrix_C_computed.sync_device();
typename IteratorA::Params params_A(matrix_A.layout());
typename IteratorB::Params params_B(matrix_B.layout());
test::gemm::threadblock::kernel_mma_planar_complex<Mma><<<grid, block>>>(
problem_size,
params_A,
matrix_A.device_data(),
matrix_A.imaginary_stride(),
params_B,
matrix_B.device_data(),
matrix_B.imaginary_stride(),
matrix_C_computed.device_data(),
matrix_C_computed.layout().stride(0),
matrix_C_computed.imaginary_stride()
);
//
// Check error code
//
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess)
<< " kernel error: " << cudaGetErrorString(result);
matrix_C_computed.sync_host();
cutlass::reference::host::GemmPlanarComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementAccumulator
>(
problem_size,
cutlass::complex<ElementAccumulator>(ElementAccumulator(1)),
matrix_A.host_ref(),
Mma::kTransformA,
matrix_B.host_ref(),
Mma::kTransformB,
cutlass::complex<ElementAccumulator>(ElementAccumulator(0)),
matrix_C_reference.host_ref(),
matrix_C_reference.host_ref()
);
bool passed = cutlass::reference::host::TensorEquals(
matrix_C_computed.host_view(),
matrix_C_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
std::ofstream output("mma_pipelined_testbed_errors.txt");
output
<< "A:\n" << matrix_A.host_view() << "\n"
<< "B:\n" << matrix_B.host_view() << "\n"
<< "Reference:\n"
<< matrix_C_reference.host_view() << "\n"
<< "Computed:\n"
<< matrix_C_computed.host_view() << "\n";
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace threadblock
} // namespace gemm
} // namespace test
test/unit/gemm/warp/CMakeLists.txt
-1
View File
@@ -30,5 +30,4 @@ cutlass_test_unit_add_executable(
wmma_sm70.cu
wmma_sm72.cu
wmma_sm75.cu
testbed.h
)
test/unit/gemm/warp/testbed.h
+494 -3
View File
@@ -102,6 +102,7 @@ __global__ void kernel(
FragmentA frag_A;
FragmentB frag_B;
FragmentC accum;
Mma mma;
@@ -306,13 +307,22 @@ struct Testbed {
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(tensor_A.host_data(), tensor_A.stride(), tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(tensor_B.host_data(), tensor_B.stride(), tensor_B.extent());
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
@@ -459,6 +469,7 @@ struct TestbedComplex {
tensor_B.host_ref(),
Mma::kTransformB,
ElementC(0),
tensor_C.host_ref(),
tensor_D_reference.host_ref()
);
@@ -486,13 +497,15 @@ struct TestbedComplex {
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical):\n" << tensor_B_physical << "\n\n";
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
@@ -506,6 +519,484 @@ struct TestbedComplex {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Test kernel
template <typename Mma, typename ThreadblockShape>
__global__ void kernel_transform(
typename Mma::ElementC *output_C,
typename Mma::ElementA const *input_A,
typename Mma::ElementB const *input_B,
typename Mma::ElementC const *input_C,
int iterations = 1) {
// Use AlignedBuffer to store trivially copyable objects in unions and __shared__ buffers.
__shared__ cutlass::AlignedBuffer<
typename Mma::ElementA, ThreadblockShape::kM * ThreadblockShape::kK> smem_buffer_A;
__shared__ cutlass::AlignedBuffer<
typename Mma::ElementB, ThreadblockShape::kN * ThreadblockShape::kK> smem_buffer_B;
if (threadIdx.x == 0) {
typename Mma::ElementA *smem_ptr_A = smem_buffer_A.data();
#pragma unroll 1
for (int i = 0; i < smem_buffer_A.size(); ++i) {
cutlass::ReferenceFactory<typename Mma::ElementA>::get(smem_ptr_A, i) =
cutlass::ReferenceFactory<typename cutlass::platform::remove_const<
typename Mma::ElementA>::type>::get(input_A, i);
}
typename Mma::ElementB *smem_ptr_B = smem_buffer_B.data();
#pragma unroll 1
for (int i = 0; i < smem_buffer_B.size(); ++i) {
cutlass::ReferenceFactory<typename Mma::ElementB>::get(smem_ptr_B, i) =
cutlass::ReferenceFactory<typename cutlass::platform::remove_const<
typename Mma::ElementB>::type>::get(input_B, i);
}
}
__syncthreads();
//
// Construct warp-level matrix product
//
using FragmentA = typename Mma::FragmentA;
using FragmentB = typename Mma::FragmentB;
using FragmentC = typename Mma::FragmentC;
using TransformedFragmentA = typename Mma::TransformedFragmentA;
using TransformedFragmentB = typename Mma::TransformedFragmentB;
typename Mma::LayoutA layout_A = Mma::LayoutA::packed({ThreadblockShape::kM, ThreadblockShape::kK});
typename Mma::LayoutB layout_B = Mma::LayoutB::packed({ThreadblockShape::kK, ThreadblockShape::kN});
typename Mma::LayoutC layout_C = Mma::LayoutC::packed({Mma::Shape::kM, Mma::Shape::kN});
typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::LaneId());
typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::LaneId());
FragmentA loaded_frag_A;
FragmentB loaded_frag_B;
TransformedFragmentA transformed_frag_A;
TransformedFragmentB transformed_frag_B;
FragmentC accum;
Mma mma;
accum.clear();
CUTLASS_PRAGMA_NO_UNROLL
for (int iter = 0; iter < iterations; ++iter) { // place in loop that is not unrolled
CUTLASS_PRAGMA_UNROLL
for (int k = 0; k < ThreadblockShape::kK;
k += Mma::Policy::MmaShape::kK) {
iter_A.load(loaded_frag_A);
iter_B.load(loaded_frag_B);
++iter_A;
++iter_B;
mma.transform(transformed_frag_A, transformed_frag_B, loaded_frag_A,
loaded_frag_B);
mma(accum, transformed_frag_A, transformed_frag_B, accum);
}
}
typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::LaneId());
iter_C.store(accum);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Warp-level matrix multiply-accumulate
typename Mma_,
/// Size of threadblock-scoped shape used to store SMEM
typename ThreadblockShape_,
/// The innter product operation performed by GEMM
typename Operator_ = cutlass::arch::OpMultiplyAdd
>
struct TransformTestbed {
/// Thread-level matrix multiply-accumulate operator
using Mma = Mma_;
using ThreadblockShape = ThreadblockShape_;
using Operator = Operator_;
using Shape = typename Mma::Shape;
using ElementA = typename Mma::ElementA;
using LayoutA = typename Mma::LayoutA;
using ElementB = typename Mma::ElementB;
using LayoutB = typename Mma::LayoutB;
using ElementC = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
//
// Data members
//
cutlass::HostTensor<ElementA, LayoutA> tensor_A;
cutlass::HostTensor<ElementB, LayoutB> tensor_B;
cutlass::HostTensor<ElementC, LayoutC> tensor_C;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
//
// Methods
//
/// Allocates workspace in device memory
TransformTestbed() {
tensor_A.reset(cutlass::make_Coord(ThreadblockShape::kM, ThreadblockShape::kK));
tensor_B.reset(cutlass::make_Coord(ThreadblockShape::kK, ThreadblockShape::kN));
tensor_C.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_computed.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
}
/// Runs the test
bool run(
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementA>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementA>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
tensor_A.host_view(), seed, scope_max, scope_min, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_A.host_data(),
tensor_A.capacity());
} else if (init_A == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementB>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementB>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
tensor_B.host_view(), seed + 16, scope_max, scope_min, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_B.host_data(),
tensor_B.capacity());
} else if (init_B == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
cutlass::reference::host::TensorFill(
tensor_C.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_computed.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_reference.host_view(),
ElementC(0)
);
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D_computed.sync_device();
// launch kernel
kernel_transform<Mma, ThreadblockShape><<<dim3(1, 1), dim3(32, 1, 1)>>>(
tensor_D_computed.device_data(), tensor_A.device_data(),
tensor_B.device_data(), tensor_C.device_data());
// verify no errors
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess) << "CUDA ERROR: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
tensor_D_computed.sync_host();
//
// Reference implementation
//
cutlass::reference::host::Gemm<ElementA, LayoutA, ElementB, LayoutB,
ElementC, LayoutC, ElementC, ElementC,
Operator>
reference_gemm;
reference_gemm(
{Shape::kM, Shape::kN, ThreadblockShape::kK},
ElementC(1),
tensor_A.host_ref(),
tensor_B.host_ref(),
ElementC(0),
tensor_D_reference.host_ref()
);
//
// Verify equivalence
//
// compare
bool passed = cutlass::reference::host::TensorEquals(
tensor_D_computed.host_view(),
tensor_D_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
<< "Reference:\n" << tensor_D_reference.host_view() << "\n\n"
<< "Computed:\n" << tensor_D_computed.host_view() << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Warp-level matrix multiply-accumulate
typename Mma_,
/// Size of threadblock-scoped shape used to store SMEM
typename ThreadblockShape_
>
struct TransformedTestbedComplex {
/// Thread-level matrix multiply-accumulate operator
using Mma = Mma_;
using ThreadblockShape = ThreadblockShape_;
using Shape = typename Mma::Shape;
using ElementA = typename Mma::ElementA;
using LayoutA = typename Mma::LayoutA;
using ElementB = typename Mma::ElementB;
using LayoutB = typename Mma::LayoutB;
using ElementC = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
//
// Data members
//
cutlass::HostTensor<ElementA, LayoutA> tensor_A;
cutlass::HostTensor<ElementB, LayoutB> tensor_B;
cutlass::HostTensor<ElementC, LayoutC> tensor_C;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
//
// Methods
//
/// Allocates workspace in device memory
TransformedTestbedComplex() {
tensor_A.reset(cutlass::make_Coord(ThreadblockShape::kM, ThreadblockShape::kK));
tensor_B.reset(cutlass::make_Coord(ThreadblockShape::kK, ThreadblockShape::kN));
tensor_C.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_computed.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
}
/// Runs the test
bool run(
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(tensor_A.host_view(),
seed, 8, -8, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_A.host_data(),
tensor_A.capacity());
} else if (init_A == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(tensor_B.host_view(),
seed + 16, 8, -8, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_B.host_data(),
tensor_B.capacity());
} else if (init_B == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
cutlass::reference::host::TensorFill(
tensor_C.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_computed.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_reference.host_view(),
ElementC(0)
);
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D_computed.sync_device();
// launch kernel
kernel_transform<Mma, ThreadblockShape><<< dim3(1, 1), dim3(32, 1, 1) >>>(
tensor_D_computed.device_data(),
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data());
// verify no errors
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess) << "CUDA ERROR: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
tensor_D_computed.sync_host();
//
// Reference implementation
//
cutlass::reference::host::GemmComplex(
{Shape::kM, Shape::kN, ThreadblockShape::kK},
ElementC(1),
tensor_A.host_ref(),
Mma::kTransformA,
tensor_B.host_ref(),
Mma::kTransformB,
ElementC(0),
tensor_C.host_ref(),
tensor_D_reference.host_ref()
);
//
// Verify equivalence
//
// compare
bool passed = cutlass::reference::host::TensorEquals(
tensor_D_computed.host_view(),
tensor_D_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
<< "Reference:\n" << tensor_D_reference.host_view() << "\n\n"
<< "Computed:\n" << tensor_D_computed.host_view() << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace warp
} // namespace gemm
} // namespace test
test/unit/nvrtc/stdlib/stdint.h
+88
View File
@@ -33,3 +33,91 @@ typedef int int32_t;
typedef unsigned int uint32_t;
typedef long long int int64_t;
typedef unsigned long long int uint64_t;
#if defined __x86_64__ && !defined __ILP32__
# define __WORDSIZE 64
#else
# define __WORDSIZE 32
#endif
/* Small types. */
/* Signed. */
typedef signed char int_least8_t;
typedef short int int_least16_t;
typedef int int_least32_t;
#if __WORDSIZE == 64
typedef long int int_least64_t;
#else
__extension__
typedef long long int int_least64_t;
#endif
/* Unsigned. */
typedef unsigned char uint_least8_t;
typedef unsigned short int uint_least16_t;
typedef unsigned int uint_least32_t;
#if __WORDSIZE == 64
typedef unsigned long int uint_least64_t;
#else
__extension__
typedef unsigned long long int uint_least64_t;
#endif
/* Fast types. */
/* Signed. */
typedef signed char int_fast8_t;
#if __WORDSIZE == 64
typedef long int int_fast16_t;
typedef long int int_fast32_t;
typedef long int int_fast64_t;
#else
typedef int int_fast16_t;
typedef int int_fast32_t;
__extension__
typedef long long int int_fast64_t;
#endif
/* Unsigned. */
typedef unsigned char uint_fast8_t;
#if __WORDSIZE == 64
typedef unsigned long int uint_fast16_t;
typedef unsigned long int uint_fast32_t;
typedef unsigned long int uint_fast64_t;
#else
typedef unsigned int uint_fast16_t;
typedef unsigned int uint_fast32_t;
__extension__
typedef unsigned long long int uint_fast64_t;
#endif
/* Types for `void *' pointers. */
#if __WORDSIZE == 64
# ifndef __intptr_t_defined
typedef long int intptr_t;
# define __intptr_t_defined
# endif
typedef unsigned long int uintptr_t;
#else
# ifndef __intptr_t_defined
typedef int intptr_t;
# define __intptr_t_defined
# endif
typedef unsigned int uintptr_t;
#endif
/* Largest integral types. */
#if __WORDSIZE == 64
typedef long int intmax_t;
typedef unsigned long int uintmax_t;
#else
__extension__
typedef long long int intmax_t;
__extension__
typedef unsigned long long int uintmax_t;
#endif