leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

CUTLASS 2.1 (#83)

Browse Source 
CUTLASS 2.1 contributes:
- BLAS-style host-side API added to CUTLASS Library
- Planar Complex GEMM kernels targeting Volta and Turing Tensor Cores
- Minor enhancements and bug fixes
This commit is contained in:
Andrew Kerr
2020-04-07 13:51:25 -07:00
committed by GitHub
parent 7c0cd26d13
commit 96dab34ad9
196 changed files with 20653 additions and 1995 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/gemm/device/CMakeLists.txt
+6 -3
View File
@@ -23,9 +23,13 @@
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device
BATCH_SOURCES ON
BATCH_SIZE 4
gemm_f16t_f16n_f16t_tensor_op_f16_sm75.cu
gemm_f16n_f16t_f16t_tensor_op_f16_sm75.cu
gemm_f16n_f16t_f16t_tensor_op_f16_sm75_slicedk.cu
gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
gemm_f16n_f16n_f16t_tensor_op_f32_sm75.cu
@@ -90,6 +94,7 @@ cutlass_test_unit_add_executable(
simt_zgemm_tn_sm50.cu
simt_zgemm_tt_sm50.cu
gemm_splitk_serial_tensor_op_sm75.cu
gemm_splitk_tensor_op_sm75.cu
gemm_splitk_tensor_op_sm70.cu
gemm_splitk_simt_sm50.cu
@@ -145,5 +150,3 @@ cutlass_test_unit_add_executable(
gemm_f16t_f16n_f32t_singlestage_wmma_tensor_op_f32_sm70.cu
)
test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_sm75_slicedk.cu → test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm75.cu
View File
test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_sm75_slicedk.cu → test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_slicedk_sm75.cu
View File
test/unit/gemm/device/gemm_planar_complex_f16_f16_f32_tensor_op_sm70.cu
+131
View File
@@ -0,0 +1,131 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-level GEMM API for Planar Complex.
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/kernel/default_gemm_planar_complex_universal.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "testbed_planar_complex.h"
#if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
using gemm_planar_complex_s884_tn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
cutlass::half_t,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm70,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
float,
4,
float,
float
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2,
cutlass::arch::OpMultiplyAdd
>::GemmKernel;
struct gemm_planar_complex_s884_tn : gemm_planar_complex_s884_tn_base {
};
////////////////////////////////////////////////////////////////////////////////
TEST(SM70_Device_GemmPlanarComplex_f16t_f16n_f32n_tensor_op_f32_884, 64x64x32_32x32x32) {
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_tn>;
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
using gemm_planar_complex_s884_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
8,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm70,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
float,
4,
float,
float
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2,
cutlass::arch::OpMultiplyAdd
>::GemmKernel;
struct gemm_planar_complex_s884_nt : gemm_planar_complex_s884_nt_base {
};
////////////////////////////////////////////////////////////////////////////////
TEST(SM70_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_884, 64x64x32_32x32x32) {
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_nt>;
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
test/unit/gemm/device/gemm_s4t_s4n_s32n_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x256x128_64x64x128) {
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 256x128x128_64x64x128) {
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x128x128_64x64x128) {
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 64x128x128_32x64x128) {
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 128x64x128_64x32x128) {
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_tensor_op_s32, 64x64x128_32x32x128) {
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32n_wmma_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -66,7 +66,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x256x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -98,7 +98,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 256x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -162,7 +162,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 64x128x128_32x64x128_8x8x
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -194,7 +194,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 128x64x128_64x32x128_8x8x
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -226,7 +226,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32n_wmma_tensor_op_s32, 64x64x128_32x32x128_8x8x3
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32t_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x256x128_64x64x128) {
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x128x128_64x64x128) {
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x128_64x64x128) {
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x128x128_32x64x128) {
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x128_64x32x128) {
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x128_32x32x128) {
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s32t_wmma_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -66,7 +66,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x256x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -98,7 +98,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 256x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x128x128_64x64x128_8x8
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -162,7 +162,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 64x128x128_32x64x128_8x8x
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -194,7 +194,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 128x64x128_64x32x128_8x8x
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -226,7 +226,7 @@ TEST(SM75_Device_Gemm_s4t_s4n_s32t_wmma_tensor_op_s32, 64x64x128_32x32x128_8x8x3
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s4t_s4n_s4n_tensor_op_s32_sm75.cu
+243
View File
@@ -0,0 +1,243 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x256x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 256x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 64x128x128_32x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 128x64x128_64x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4n_tensor_op_s32, 64x64x128_32x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s4t_s4n_s4t_tensor_op_s32_sm75.cu
+243
View File
@@ -0,0 +1,243 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x256x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 256x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x128x128_64x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 128>,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 64x128x128_32x64x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 128>,
cutlass::gemm::GemmShape<32, 64, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 128x64x128_64x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 128>,
cutlass::gemm::GemmShape<64, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
TEST(SM75_Device_Gemm_s4t_s4n_s4t_tensor_op_s32, 64x64x128_32x32x128) {
using ElementOutput = cutlass::int4b_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
cutlass::int4b_t,
cutlass::layout::RowMajor,
cutlass::int4b_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 128>,
cutlass::gemm::GemmShape<32, 32, 128>,
cutlass::gemm::GemmShape<8, 8, 32>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8n_s8t_s8n_tensor_op_s32_sm75.cu
+14 -28
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 32x64x64_16x32x64) {
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -101,11 +99,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -137,11 +133,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -173,11 +167,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -209,11 +201,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -245,11 +235,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -281,11 +269,9 @@ TEST(SM75_Device_Gemm_s8n_s8t_s8n_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
64 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
64 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_s8t_s8n_s32n_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32n_wmma_tensor_op_s32_sm72.cu
+3 -3
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -96,7 +96,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -130,7 +130,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32n_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32t_tensor_op_s32_sm75.cu
+6 -6
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x256x64_64x64x64) {
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x128x64_64x64x64) {
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -129,7 +129,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x64_64x64x64) {
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -161,7 +161,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x128x64_32x64x64) {
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -193,7 +193,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x64_64x32x64) {
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -225,7 +225,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x64_32x32x64) {
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s32t_wmma_tensor_op_s32_sm72.cu
+4 -4
View File
@@ -65,7 +65,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -97,7 +97,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -131,7 +131,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
@@ -165,7 +165,7 @@ TEST(SM75_Device_Gemm_s8t_s8n_s32t_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombination<
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
test/unit/gemm/device/gemm_s8t_s8n_s8n_tensor_op_s32_sm75.cu
+75 -13
View File
@@ -57,9 +57,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x256x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -77,11 +76,10 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 256x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
@@ -96,9 +94,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -116,15 +113,80 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 64x128x64_32x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 128x64x64_64x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t,
cutlass::layout::RowMajor,
int8_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8n_tensor_op_s32, 64x64x64_32x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t,
cutlass::layout::RowMajor,
int8_t,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput,
32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementCompute
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8t_s8n_s8n_wmma_tensor_op_s32_sm72.cu
+8 -16
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16x
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -96,11 +94,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -130,11 +126,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -164,11 +158,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8n_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_s8t_s8n_s8t_tensor_op_s32_sm75.cu
+50 -12
View File
@@ -57,9 +57,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x256x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -76,9 +75,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 256x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<256, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -95,9 +93,8 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x128x64_64x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 128, 64>,
cutlass::gemm::GemmShape<64, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
@@ -115,14 +112,55 @@ CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 64x128x64_32x64x64,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 128x64x64_64x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t, cutlass::layout::RowMajor, int8_t, cutlass::layout::ColumnMajor,
ElementOutput, cutlass::layout::RowMajor, ElementAccumulator,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 64, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
test::gemm::device::Testbed<Gemm> testbed;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
CUTLASS_TEST_L0(SM75_Device_Gemm_s8t_s8n_s8t_tensor_op_s32, 64x64x64_32x32x64, {
using ElementOutput = int8_t;
using ElementAccumulator = int32_t;
using ElementCompute = float;
using Gemm = cutlass::gemm::device::Gemm<
int8_t, cutlass::layout::RowMajor, int8_t, cutlass::layout::ColumnMajor,
ElementOutput, cutlass::layout::RowMajor, ElementAccumulator,
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm75,
cutlass::gemm::GemmShape<64, 64, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<8, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 32 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 2>;
test::gemm::device::Testbed<Gemm> testbed;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
} )
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_s8t_s8n_s8t_wmma_tensor_op_s32_sm72.cu
+8 -16
View File
@@ -65,11 +65,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 128x128x32_64x64x32_16x16x
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -97,11 +95,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x32x64_16x16x1
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -131,11 +127,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x64x64_32x8x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<32, 8, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
@@ -165,11 +159,9 @@ TEST(SM75_Device_Gemm_s8t_s8n_s8t_wmma_tensor_op_s32, 64x128x64_32x64x64_8x32x16
cutlass::gemm::GemmShape<64, 128, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<8, 32, 16>,
cutlass::epilogue::thread::LinearCombinationClamp<
cutlass::epilogue::thread::FastLinearCombinationClamp<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
128 / cutlass::sizeof_bits<ElementOutput>::value
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
2
test/unit/gemm/device/gemm_splitk_serial_tensor_op_sm75.cu
+107
View File
@@ -0,0 +1,107 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm.h"
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/gemm.h"
#include "testbed.h"
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM75_Device_GemmSplitKSerial_f16n_f16n_f16t_tensor_op_f32, 128x256x32_64x64x32) {
using ElementA = cutlass::half_t;
using ElementB = cutlass::half_t;
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
static const int kStages = 2;
static const int kAlignmentA = cutlass::gemm::device::DefaultGemmConfiguration<
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
ElementA,
ElementB,
ElementOutput,
ElementAccumulator>::kAlignmentA;
static const int kAlignmentB = cutlass::gemm::device::DefaultGemmConfiguration<
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
ElementA,
ElementB,
ElementOutput,
ElementAccumulator>::kAlignmentB;
static const bool kSplitKSerial = true;
using Gemm = cutlass::gemm::device::Gemm<
ElementA,
cutlass::layout::ColumnMajor,
ElementB,
cutlass::layout::ColumnMajor,
ElementOutput,
cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm75,
cutlass::gemm::GemmShape<128, 256, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 8>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,
kStages,
kAlignmentA,
kAlignmentB,
kSplitKSerial
>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif
test/unit/gemm/device/gemm_splitk_tensor_op_sm70.cu
+1
View File
@@ -42,6 +42,7 @@
#include "testbed_splitk.h"
// These tests cannot run unless CUDA 10.1 Toolkit or later is used.
#if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/gemm_splitk_tensor_op_sm75.cu
+1
View File
@@ -42,6 +42,7 @@
#include "testbed_splitk.h"
// These tests cannot run unless CUDA 10.2 Toolkit or later is used.
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed.h
+16 -26
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
@@ -41,20 +43,7 @@
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/host/gemm.h"
inline char const *to_string(cutlass::Status status) {
switch (status) {
case cutlass::Status::kSuccess: return "kSuccess";
case cutlass::Status::kErrorMisalignedOperand: return "kErrorMisalignedOperand";
case cutlass::Status::kErrorInvalidLayout: return "kErrorInvalidLayout";
case cutlass::Status::kErrorInvalidProblem: return "kErrorInvalidProblem";
case cutlass::Status::kErrorNotSupported: return "kErrorNotSupported";
case cutlass::Status::kErrorWorkspaceNull: return "kErrorWorkspaceNull";
case cutlass::Status::kErrorInternal: return "kErrorInternal";
case cutlass::Status::kInvalid: return "kInvalid";
}
return "invalid";
}
#include "testbed_utils.h"
namespace test {
namespace gemm {
@@ -185,9 +174,12 @@ struct Testbed {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
if (tensor_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
if (reference_D.size() > 1)
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
@@ -341,18 +333,12 @@ bool TestAllGemm() {
(cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::RowMajor>::value ||
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::ColumnMajor>::value) ? 4 : kAlignment;
int problem_size_m[] = {
kAlignmentM, 512 - 3*kAlignmentM
};
int problem_size_m[] = {kAlignmentM, 512 - 3 * kAlignmentM};
int problem_size_n[] = {
kAlignmentN, 512 - 2*kAlignmentN
};
int problem_size_n[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int problem_size_k[] = {
kAlignmentK, Gemm::ThreadblockShape::kK * Gemm::kStages - kAlignmentK
};
kAlignmentK, Gemm::ThreadblockShape::kK * (Gemm::kStages + 1) - kAlignmentK};
int split_k_slices[] = {
1, 2, 3
@@ -379,6 +365,10 @@ bool TestAllGemm() {
continue;
}
if (split_k > 1 && k / Gemm::ThreadblockShape::kK < split_k) {
continue;
}
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
test/unit/gemm/device/testbed_complex.h
+3
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
@@ -90,6 +92,7 @@ struct TestbedComplex : public Testbed<Gemm> {
this->tensor_B.host_ref(),
Gemm::kTransformB,
beta,
this->tensor_C.host_ref(),
this->reference_D.host_ref(),
ElementAccumulator(0)
);
test/unit/gemm/device/testbed_planar_complex.h
+283
View File
@@ -0,0 +1,283 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/gemm_planar_complex.h"
#include "cutlass/util/host_tensor_planar_complex.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
class TestbedPlanarComplex {
public:
using ElementA = typename Gemm::ElementA;
using LayoutA = typename Gemm::LayoutA;
using ElementB = typename Gemm::ElementB;
using LayoutB = typename Gemm::LayoutB;
using ElementC = typename Gemm::ElementC;
using LayoutC = typename Gemm::LayoutC;
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
using ElementAccumulator = typename Gemm::ElementAccumulator;
//
// Data members
//
cutlass::gemm::GemmCoord problem_size;
cutlass::HostTensorPlanarComplex<ElementA, LayoutA> tensor_A;
cutlass::HostTensorPlanarComplex<ElementB, LayoutB> tensor_B;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_C;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_D;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> tensor_D_ref;
//
// Methods
//
TestbedPlanarComplex(cutlass::gemm::GemmCoord const & problem_size): problem_size(problem_size) {
tensor_A.reset({problem_size.m(), problem_size.k()});
tensor_B.reset({problem_size.k(), problem_size.n()});
tensor_C.reset({problem_size.m(), problem_size.n()});
tensor_D.reset({problem_size.m(), problem_size.n()});
tensor_D_ref.reset({problem_size.m(), problem_size.n()}, false);
}
void initialize() {
uint64_t seed = 1073;
int scope_max = 8;
int scope_min = -8;
cutlass::reference::host::TensorFillRandomUniform(
tensor_A.host_view(), seed, scope_max, scope_min, 0);
cutlass::reference::host::TensorFillRandomUniform(
tensor_B.host_view(), seed * 2019, scope_max, scope_min, 0);
cutlass::reference::host::TensorFillRandomUniform(
tensor_C.host_view(), seed * 2020, scope_max, scope_min, 0);
cutlass::reference::host::TensorFill(tensor_D.host_view());
cutlass::reference::host::TensorFill(tensor_D_ref.host_view());
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D.sync_device();
}
bool run(
cutlass::complex<ElementCompute> alpha = {1, 0},
cutlass::complex<ElementCompute> beta = {0, 0}) {
initialize();
int batch_count = 1;
ElementA *ptr_A = tensor_A.device_data();
ElementB *ptr_B = tensor_B.device_data();
ElementC *ptr_C = tensor_C.device_data();
ElementC *ptr_D = tensor_D.device_data();
int lda = tensor_A.layout().stride(0);
int ldb = tensor_B.layout().stride(0);
int ldc = tensor_C.layout().stride(0);
int ldd = tensor_D.layout().stride(0);
int64_t imag_stride_A = tensor_A.imaginary_stride();
int64_t imag_stride_B = tensor_B.imaginary_stride();
int64_t imag_stride_C = tensor_C.imaginary_stride();
int64_t imag_stride_D = tensor_D.imaginary_stride();
//
// Launch device kernel
//
Gemm gemm_op;
typename Gemm::Arguments args{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
batch_count,
{alpha, beta},
ptr_A,
ptr_A + imag_stride_A,
ptr_B,
ptr_B + imag_stride_B,
ptr_C,
ptr_C + imag_stride_C,
ptr_D,
ptr_D + imag_stride_D,
lda,
lda,
ldb,
ldb,
ldc,
ldc,
ldd,
ldd
};
cutlass::Status status = gemm_op(args);
EXPECT_EQ(status, cutlass::Status::kSuccess);
cudaError_t error = cudaDeviceSynchronize();
tensor_D.sync_host();
//
// Compute reference
//
cutlass::reference::host::GemmPlanarComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementAccumulator
>(
problem_size,
alpha,
tensor_A.host_ref(),
Gemm::kTransformA,
tensor_B.host_ref(),
Gemm::kTransformB,
beta,
tensor_C.host_ref(),
tensor_D_ref.host_ref()
);
bool passed = cutlass::reference::host::TensorEquals(
tensor_D.host_view(),
tensor_D_ref.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
std::ofstream output("gemm_planar_complex.txt");
output
<< "A:\n" << tensor_A.host_view() << "\n"
<< "B:\n" << tensor_B.host_view() << "\n"
<< "C:\n" << tensor_C.host_view() << "\n"
<< "Reference:\n"
<< tensor_D_ref.host_view() << "\n"
<< "Computed:\n"
<< tensor_D.host_view() << "\n";
}
return passed;
}
};
template <typename Gemm>
bool TestOneGemmPlanarComplex(cutlass::gemm::GemmCoord problem_size) {
TestbedPlanarComplex<Gemm> testbed(problem_size);
return testbed.run();
}
template <typename Gemm>
bool TestAllGemmPlanarComplex() {
int M[] = {
16, 264,
};
int N[] = {
16, 248,
};
int K[] = {
8, 96,
};
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
cutlass::complex<ElementCompute> alpha_values[] = {
{ElementCompute(1.25), ElementCompute(-0.5)}
};
cutlass::complex<ElementCompute> beta_values[] = {
{ElementCompute(-2.25), ElementCompute(1.5)}
};
for (int m : M) {
for (int n : N) {
for (int k : K) {
test::gemm::device::TestbedPlanarComplex<Gemm> testbed({m, n, k});
for (auto const &alpha : alpha_values) {
for (auto const &beta : beta_values) {
bool passed = testbed.run(alpha, beta);
if (!passed) {
return false;
}
}
}
}
}
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace gemm
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed_splitk.h
+2
View File
@@ -26,6 +26,8 @@
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
test/unit/gemm/device/testbed_universal.h
+480
View File
@@ -0,0 +1,480 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include <fstream>
#include <sstream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/gemm_complex.h"
#include "testbed_utils.h"
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
struct TestbedUniversal {
using ElementAccumulator = typename Gemm::ElementAccumulator;
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
/// Initialization
cutlass::Distribution::Kind init_A;
cutlass::Distribution::Kind init_B;
cutlass::Distribution::Kind init_C;
uint64_t seed;
cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A;
cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_C;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_D;
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> reference_D;
//
// Methods
//
TestbedUniversal(
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint64_t seed_ = 2080
):
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
/// Helper to initialize a tensor view
template <typename Element, typename Layout>
bool initialize_tensor(
cutlass::TensorView<Element, Layout> view,
cutlass::Distribution::Kind dist_kind,
uint64_t seed) {
if (dist_kind == cutlass::Distribution::Uniform) {
double scope_max, scope_min;
int bits_input = cutlass::sizeof_bits<Element>::value;
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
} else if (bits_output == 16) {
scope_max = 5;
scope_min = -5;
} else {
scope_max = 8;
scope_min = -8;
}
cutlass::reference::host::TensorFillRandomUniform(
view, seed, scope_max, scope_min, 0);
}
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
}
else if (dist_kind == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(
view.data(), view.capacity());
}
else {
// TODO: Implement the rest
EXPECT_TRUE(false) << "Not implemented";
return false;
}
return true;
}
/// Initializes data structures
void initialize(cutlass::gemm::GemmCoord problem_size) {
//
// Allocate the GEMM workspace
//
tensor_A.resize(problem_size.mk());
tensor_B.resize(problem_size.kn());
tensor_C.resize(problem_size.mn());
tensor_D.resize(problem_size.mn());
reference_D.resize(problem_size.mn(), false);
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
// It is possible to randomly initialize to all zeros, so override this with non-zeros
// in the upper left corner of each operand.
tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D.sync_device();
}
/// Compares computed reference with device reference and outputs to a file if incorrect
bool compare_reference(
cutlass::gemm::GemmCoord problem_size,
ElementCompute alpha,
ElementCompute beta) {
tensor_D.sync_host();
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
EXPECT_TRUE(passed) << " mismatched reference";
if (!passed) {
/*
std::stringstream fname;
fname << "error_Gemm_device_"
<< problem_size.m() << "x"
<< problem_size.n() << "x"
<< problem_size.k() << "_"
<< Gemm::ThreadblockShape::kM << "x"
<< Gemm::ThreadblockShape::kN << "x"
<< Gemm::ThreadblockShape::kK << "_"
<< Gemm::WarpShape::kM << "x"
<< Gemm::WarpShape::kN << "x"
<< Gemm::WarpShape::kK << ".txt";
std::ofstream file(fname.str());
*/
std::ofstream file("testbed_universal_errors.txt");
file
<< "problem: " << problem_size
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
file
<< "A =\n" << tensor_A.host_view()
<< "\nB =\n" << tensor_B.host_view()
<< "\nC =\n" << tensor_C.host_view()
<< "\n\nReference =\n" << reference_D.host_view()
<< "\nComputed =\n" << tensor_D.host_view();
}
return passed;
}
/// Verifies the result is a GEMM
bool verify(
cutlass::gemm::GemmCoord problem_size,
ElementCompute alpha,
ElementCompute beta) {
//
// Verify
//
cutlass::reference::host::GemmComplex<
typename Gemm::ElementA, typename Gemm::LayoutA,
typename Gemm::ElementB, typename Gemm::LayoutB,
typename Gemm::ElementC, typename Gemm::LayoutC,
ElementCompute, ElementAccumulator
>(
problem_size,
alpha,
tensor_A.host_ref(),
Gemm::kTransformA,
tensor_B.host_ref(),
Gemm::kTransformB,
beta,
tensor_C.host_ref(),
reference_D.host_ref(),
ElementAccumulator(0)
);
return compare_reference(problem_size, alpha, beta);
}
/// Executes one test
bool run(
cutlass::gemm::GemmUniversalMode mode,
cutlass::gemm::GemmCoord problem_size,
int batch_count = 1,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(0)) {
this->initialize(problem_size);
//
// Initialize the GEMM operator
//
typename Gemm::Arguments arguments{
mode,
problem_size,
batch_count,
{alpha, beta},
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data(),
tensor_D.device_data(),
problem_size.m() * problem_size.k(),
problem_size.n() * problem_size.k(),
problem_size.m() * problem_size.n(),
problem_size.m() * problem_size.n(),
tensor_A.layout().stride(0),
tensor_B.layout().stride(0),
tensor_C.layout().stride(0),
tensor_D.layout().stride(0)
};
Gemm gemm_op;
size_t workspace_size = Gemm::get_workspace_size(arguments);
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
//
// Run the GEMM
//
status = gemm_op();
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
//
// Verify
//
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
bool TestGemmUniversal(
cutlass::gemm::GemmCoord const & problem_size,
cutlass::gemm::GemmUniversalMode mode,
int batch_count,
double alpha = 1.0,
double beta = 2.0) {
bool passed = true;
TestbedUniversal<Gemm> testbed;
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
passed = testbed.run(
mode,
problem_size,
batch_count,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
return passed;
}
template <typename Gemm>
bool TestAllGemmUniversal() {
bool passed = true;
int const kMinimumOperandElementSize =
std::min(
int(cutlass::sizeof_bits<typename Gemm::ElementA>::value),
int(cutlass::sizeof_bits<typename Gemm::ElementB>::value));
int const kAlignment = cutlass::platform::is_same<
typename Gemm::OperatorClass,
cutlass::arch::OpClassSimt>::value ? 1 : 128 / kMinimumOperandElementSize;
// int8_t gemm alignment constraints
int const kAlignmentM = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementA, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::ColumnMajor>::value ? 4 : kAlignment;
int const kAlignmentN = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementB, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::RowMajor>::value ? 4 : kAlignment;
int const kAlignmentK = cutlass::platform::is_same<typename Gemm::OperatorClass, cutlass::arch::OpClassSimt>::value &&
cutlass::platform::is_same<typename Gemm::ElementA, int8_t>::value &&
cutlass::platform::is_same<typename Gemm::ElementB, int8_t>::value &&
(cutlass::platform::is_same<typename Gemm::LayoutA, cutlass::layout::RowMajor>::value ||
cutlass::platform::is_same<typename Gemm::LayoutB, cutlass::layout::ColumnMajor>::value) ? 4 : kAlignment;
cutlass::gemm::GemmUniversalMode modes[] = {
cutlass::gemm::GemmUniversalMode::kGemm,
};
int problem_size_m[] = {
kAlignmentM, 512 - 3*kAlignmentM
};
int problem_size_n[] = {
kAlignmentN, 512 - 2*kAlignmentN
};
int problem_size_k[] = {
kAlignmentK,
Gemm::ThreadblockShape::kK * Gemm::kStages - kAlignmentK,
Gemm::ThreadblockShape::kK * Gemm::kStages * 3 - kAlignmentK
};
int batch_counts[] = { // may be interpretted as batch count or split-K slices
1, 2, 3, 5, 7
};
double problem_alpha[] = {
1
};
double problem_beta[] = {
2.0
};
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
for (cutlass::gemm::GemmUniversalMode mode : modes) {
for (int m : problem_size_m) {
for (int n : problem_size_n) {
for (int k : problem_size_k) {
for (int batch_count : batch_counts) {
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
if (mode == cutlass::gemm::GemmUniversalMode::kGemm ||
mode == cutlass::gemm::GemmUniversalMode::kGemmSplitKParallel) {
// skip very small K problems
if (k / batch_count < 2 * Gemm::ThreadblockShape::kK) {
continue;
}
}
cutlass::gemm::GemmCoord problem_size(m, n, k);
TestbedUniversal<Gemm> testbed;
passed = testbed.run(
mode,
problem_size,
batch_count,
cutlass::from_real<ElementCompute>(alpha),
cutlass::from_real<ElementCompute>(beta)
);
if (!passed) {
return false;
}
}
}
}
}
}
}
}
/*
// large problem with high coverage
for (int split_k_slices = 1; split_k_slices <= 3; ++split_k_slices) {
TestbedUniversal<Gemm> testbed;
cutlass::gemm::GemmCoord problem_size(72, 56, 8192);
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
split_k_slices,
cutlass::from_real<ElementCompute>(1.0),
cutlass::from_real<ElementCompute>(2.0)
);
if (!passed) {
break;
}
}
*/
return passed;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace gemm
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed_utils.h
+46
View File
@@ -0,0 +1,46 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include "cutlass/cutlass.h"
inline char const *to_string(cutlass::Status status) {
switch (status) {
case cutlass::Status::kSuccess: return "kSuccess";
case cutlass::Status::kErrorMisalignedOperand: return "kErrorMisalignedOperand";
case cutlass::Status::kErrorInvalidLayout: return "kErrorInvalidLayout";
case cutlass::Status::kErrorInvalidProblem: return "kErrorInvalidProblem";
case cutlass::Status::kErrorNotSupported: return "kErrorNotSupported";
case cutlass::Status::kErrorWorkspaceNull: return "kErrorWorkspaceNull";
case cutlass::Status::kErrorInternal: return "kErrorInternal";
case cutlass::Status::kInvalid: return "kInvalid";
}
return "invalid";
}
test/unit/gemm/threadblock/mma_planar_complex_testbed.h
+345
View File
@@ -0,0 +1,345 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
*modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice,
*this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
*notice, this list of conditions and the following disclaimer in the
*documentation and/or other materials provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its
*contributors may be used to endorse or promote products derived from this
*software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
*AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
*IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
*DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY DIRECT,
*INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
*DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
*OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TOR (INCLUDING
*NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
*EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Unit testbed for kernel-level GEMM
*/
#pragma once
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/platform/platform.h"
#include "cutlass/aligned_buffer.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/layout/vector.h"
#include "cutlass/numeric_types.h"
#include "cutlass/core_io.h"
#include "cutlass/util/host_tensor_planar_complex.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/distribution.h"
#include "cutlass/util/reference/host/gemm_planar_complex.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_fill.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace threadblock {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Mma>
__global__ void kernel_mma_planar_complex(
cutlass::gemm::GemmCoord problem_size,
typename Mma::IteratorA::Params params_A,
typename Mma::IteratorA::Element *ptr_A,
int64_t imaginary_stride_A,
typename Mma::IteratorB::Params params_B,
typename Mma::IteratorB::Element *ptr_B,
int64_t imaginary_stride_B,
typename Mma::ElementC *ptr_C, int ldc, int64_t imaginary_stride_C) {
// Shared storage needed by threadblock-scoped matrix multiply-accumulate
__shared__ typename Mma::SharedStorage shared_storage;
// Compute threadblock location
cutlass::gemm::GemmCoord tb_tile_offset = {int(blockIdx.x), int(blockIdx.y),
0};
cutlass::MatrixCoord tb_offset_A{tb_tile_offset.m() * Mma::Shape::kM,
tb_tile_offset.k()};
cutlass::MatrixCoord tb_offset_B{tb_tile_offset.k(),
tb_tile_offset.n() * Mma::Shape::kN};
// Compute position within threadblock
int tb_thread_id = threadIdx.y * blockDim.x + threadIdx.x;
// Construct iterators to A operand
typename Mma::IteratorA iterator_A_real(params_A, ptr_A,
{problem_size.m(), problem_size.k()},
tb_thread_id, tb_offset_A);
typename Mma::IteratorA iterator_A_imag(params_A, ptr_A + imaginary_stride_A,
{problem_size.m(), problem_size.k()},
tb_thread_id, tb_offset_A);
// Construct iterators to B operand
typename Mma::IteratorB iterator_B_real(params_B, ptr_B,
{problem_size.k(), problem_size.n()},
tb_thread_id, tb_offset_B);
typename Mma::IteratorB iterator_B_imag(params_B, ptr_B + imaginary_stride_B,
{problem_size.k(), problem_size.n()},
tb_thread_id, tb_offset_B);
int warp_id = threadIdx.y;
int lane_id = threadIdx.x;
// Construct thread-scoped matrix multiply
Mma mma(shared_storage, tb_thread_id, warp_id, threadIdx.x);
typename Mma::FragmentC accum;
accum.clear();
int gemm_k_iterations = (problem_size.k() + Mma::Shape::kK - 1) / Mma::Shape::kK;
// Compute threadblock-scoped matrix multiply-add
mma(gemm_k_iterations, accum, iterator_A_real, iterator_A_imag, iterator_B_real, iterator_B_imag, accum);
// Output results
typename Mma::Operator::IteratorC iterator_C({ptr_C, ldc}, lane_id);
iterator_C.add_tile_offset(
{(tb_tile_offset.m() * Mma::WarpCount::kM) +
(warp_id % Mma::WarpCount::kM),
(tb_tile_offset.n() * Mma::WarpCount::kN) +
(warp_id / Mma::WarpCount::kM)});
iterator_C.store(accum.real);
iterator_C.store_with_pointer_offset(accum.imag, imaginary_stride_C);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Threadblock-level matrix multiply-accumulate
typename Mma_>
struct TestbedPlanarComplex {
using Mma = Mma_;
using ThreadblockShape = typename Mma::Shape;
using IteratorA = typename Mma::IteratorA;
using ElementA = typename Mma::IteratorA::Element;
using LayoutA = typename Mma::IteratorA::Layout;
using IteratorB = typename Mma::IteratorB;
using ElementB = typename Mma::IteratorB::Element;
using LayoutB = typename Mma::IteratorB::Layout;
using ElementC = typename Mma::ElementC;
using ElementAccumulator = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
using ThreadMapA = typename Mma::IteratorA::ThreadMap;
using ThreadMapB = typename Mma::IteratorB::ThreadMap;
using AccessTypeA = cutlass::Array<ElementA, ThreadMapA::kElementsPerAccess>;
using AccessTypeB = cutlass::Array<ElementB, ThreadMapB::kElementsPerAccess>;
static int const Stages = Mma::kStages;
static cutlass::arch::CacheOperation::Kind const CacheOpA =
Mma::kCacheOpA;
static cutlass::arch::CacheOperation::Kind const CacheOpB =
Mma::kCacheOpB;
//
// Data members
//
cutlass::HostTensorPlanarComplex<ElementA, LayoutA> matrix_A;
cutlass::HostTensorPlanarComplex<ElementB, LayoutB> matrix_B;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> matrix_C_computed;
cutlass::HostTensorPlanarComplex<ElementC, LayoutC> matrix_C_reference;
cutlass::gemm::GemmCoord problem_size;
//
// Methods
//
/// Allocates workspace in device memory
TestbedPlanarComplex(int m, int n, int k)
: problem_size(m, n, k) {
matrix_A.reset(cutlass::make_Coord(m, k));
matrix_B.reset(cutlass::make_Coord(k, n));
matrix_C_computed.reset(cutlass::make_Coord(m, n));
matrix_C_reference.reset(cutlass::make_Coord(m, n), false);
}
/// Runs the test
bool run(
dim3 grid, dim3 block,
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementA>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementA>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
matrix_A.host_view(), seed, scope_max, scope_min, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
for (int i = 0; i < matrix_A.capacity() * 2; ++i) {
matrix_A.host_data()[i] = cutlass::half_t(float(i % 5) - 2);
}
/*
cutlass::reference::host::BlockFillSequential(matrix_A.host_data(),
matrix_A.capacity() * 2);
*/
} else if (init_A == cutlass::Distribution::Identity) {
//cutlass::reference::host::TensorFillIdentity(matrix_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementB>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementB>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
matrix_B.host_view(), seed + 16, scope_max, scope_min, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(matrix_B.host_data(),
matrix_B.capacity() * 2);
for (int i = 0; i < matrix_B.capacity() * 2; ++i) {
matrix_B.host_data()[i] = cutlass::half_t(float((i + 3) % 5) - 2);
}
} else if (init_B == cutlass::Distribution::Identity) {
//cutlass::reference::host::TensorFillIdentity(matrix_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
matrix_A.sync_device();
matrix_B.sync_device();
matrix_C_computed.sync_device();
typename IteratorA::Params params_A(matrix_A.layout());
typename IteratorB::Params params_B(matrix_B.layout());
test::gemm::threadblock::kernel_mma_planar_complex<Mma><<<grid, block>>>(
problem_size,
params_A,
matrix_A.device_data(),
matrix_A.imaginary_stride(),
params_B,
matrix_B.device_data(),
matrix_B.imaginary_stride(),
matrix_C_computed.device_data(),
matrix_C_computed.layout().stride(0),
matrix_C_computed.imaginary_stride()
);
//
// Check error code
//
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess)
<< " kernel error: " << cudaGetErrorString(result);
matrix_C_computed.sync_host();
cutlass::reference::host::GemmPlanarComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementAccumulator
>(
problem_size,
cutlass::complex<ElementAccumulator>(ElementAccumulator(1)),
matrix_A.host_ref(),
Mma::kTransformA,
matrix_B.host_ref(),
Mma::kTransformB,
cutlass::complex<ElementAccumulator>(ElementAccumulator(0)),
matrix_C_reference.host_ref(),
matrix_C_reference.host_ref()
);
bool passed = cutlass::reference::host::TensorEquals(
matrix_C_computed.host_view(),
matrix_C_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
std::ofstream output("mma_pipelined_testbed_errors.txt");
output
<< "A:\n" << matrix_A.host_view() << "\n"
<< "B:\n" << matrix_B.host_view() << "\n"
<< "Reference:\n"
<< matrix_C_reference.host_view() << "\n"
<< "Computed:\n"
<< matrix_C_computed.host_view() << "\n";
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace threadblock
} // namespace gemm
} // namespace test
test/unit/gemm/warp/CMakeLists.txt
-1
View File
@@ -30,5 +30,4 @@ cutlass_test_unit_add_executable(
wmma_sm70.cu
wmma_sm72.cu
wmma_sm75.cu
testbed.h
)
test/unit/gemm/warp/testbed.h
+494 -3
View File
@@ -102,6 +102,7 @@ __global__ void kernel(
FragmentA frag_A;
FragmentB frag_B;
FragmentC accum;
Mma mma;
@@ -306,13 +307,22 @@ struct Testbed {
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(tensor_A.host_data(), tensor_A.stride(), tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(tensor_B.host_data(), tensor_B.stride(), tensor_B.extent());
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
@@ -459,6 +469,7 @@ struct TestbedComplex {
tensor_B.host_ref(),
Mma::kTransformB,
ElementC(0),
tensor_C.host_ref(),
tensor_D_reference.host_ref()
);
@@ -486,13 +497,15 @@ struct TestbedComplex {
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical):\n" << tensor_B_physical << "\n\n";
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
@@ -506,6 +519,484 @@ struct TestbedComplex {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Test kernel
template <typename Mma, typename ThreadblockShape>
__global__ void kernel_transform(
typename Mma::ElementC *output_C,
typename Mma::ElementA const *input_A,
typename Mma::ElementB const *input_B,
typename Mma::ElementC const *input_C,
int iterations = 1) {
// Use AlignedBuffer to store trivially copyable objects in unions and __shared__ buffers.
__shared__ cutlass::AlignedBuffer<
typename Mma::ElementA, ThreadblockShape::kM * ThreadblockShape::kK> smem_buffer_A;
__shared__ cutlass::AlignedBuffer<
typename Mma::ElementB, ThreadblockShape::kN * ThreadblockShape::kK> smem_buffer_B;
if (threadIdx.x == 0) {
typename Mma::ElementA *smem_ptr_A = smem_buffer_A.data();
#pragma unroll 1
for (int i = 0; i < smem_buffer_A.size(); ++i) {
cutlass::ReferenceFactory<typename Mma::ElementA>::get(smem_ptr_A, i) =
cutlass::ReferenceFactory<typename cutlass::platform::remove_const<
typename Mma::ElementA>::type>::get(input_A, i);
}
typename Mma::ElementB *smem_ptr_B = smem_buffer_B.data();
#pragma unroll 1
for (int i = 0; i < smem_buffer_B.size(); ++i) {
cutlass::ReferenceFactory<typename Mma::ElementB>::get(smem_ptr_B, i) =
cutlass::ReferenceFactory<typename cutlass::platform::remove_const<
typename Mma::ElementB>::type>::get(input_B, i);
}
}
__syncthreads();
//
// Construct warp-level matrix product
//
using FragmentA = typename Mma::FragmentA;
using FragmentB = typename Mma::FragmentB;
using FragmentC = typename Mma::FragmentC;
using TransformedFragmentA = typename Mma::TransformedFragmentA;
using TransformedFragmentB = typename Mma::TransformedFragmentB;
typename Mma::LayoutA layout_A = Mma::LayoutA::packed({ThreadblockShape::kM, ThreadblockShape::kK});
typename Mma::LayoutB layout_B = Mma::LayoutB::packed({ThreadblockShape::kK, ThreadblockShape::kN});
typename Mma::LayoutC layout_C = Mma::LayoutC::packed({Mma::Shape::kM, Mma::Shape::kN});
typename Mma::IteratorA iter_A({smem_buffer_A.data(), layout_A}, cutlass::LaneId());
typename Mma::IteratorB iter_B({smem_buffer_B.data(), layout_B}, cutlass::LaneId());
FragmentA loaded_frag_A;
FragmentB loaded_frag_B;
TransformedFragmentA transformed_frag_A;
TransformedFragmentB transformed_frag_B;
FragmentC accum;
Mma mma;
accum.clear();
CUTLASS_PRAGMA_NO_UNROLL
for (int iter = 0; iter < iterations; ++iter) { // place in loop that is not unrolled
CUTLASS_PRAGMA_UNROLL
for (int k = 0; k < ThreadblockShape::kK;
k += Mma::Policy::MmaShape::kK) {
iter_A.load(loaded_frag_A);
iter_B.load(loaded_frag_B);
++iter_A;
++iter_B;
mma.transform(transformed_frag_A, transformed_frag_B, loaded_frag_A,
loaded_frag_B);
mma(accum, transformed_frag_A, transformed_frag_B, accum);
}
}
typename Mma::IteratorC iter_C({output_C, layout_C}, cutlass::LaneId());
iter_C.store(accum);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Warp-level matrix multiply-accumulate
typename Mma_,
/// Size of threadblock-scoped shape used to store SMEM
typename ThreadblockShape_,
/// The innter product operation performed by GEMM
typename Operator_ = cutlass::arch::OpMultiplyAdd
>
struct TransformTestbed {
/// Thread-level matrix multiply-accumulate operator
using Mma = Mma_;
using ThreadblockShape = ThreadblockShape_;
using Operator = Operator_;
using Shape = typename Mma::Shape;
using ElementA = typename Mma::ElementA;
using LayoutA = typename Mma::LayoutA;
using ElementB = typename Mma::ElementB;
using LayoutB = typename Mma::LayoutB;
using ElementC = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
//
// Data members
//
cutlass::HostTensor<ElementA, LayoutA> tensor_A;
cutlass::HostTensor<ElementB, LayoutB> tensor_B;
cutlass::HostTensor<ElementC, LayoutC> tensor_C;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
//
// Methods
//
/// Allocates workspace in device memory
TransformTestbed() {
tensor_A.reset(cutlass::make_Coord(ThreadblockShape::kM, ThreadblockShape::kK));
tensor_B.reset(cutlass::make_Coord(ThreadblockShape::kK, ThreadblockShape::kN));
tensor_C.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_computed.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
}
/// Runs the test
bool run(
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementA>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementA>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
tensor_A.host_view(), seed, scope_max, scope_min, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_A.host_data(),
tensor_A.capacity());
} else if (init_A == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
int scope_max = 8;
int scope_min = -8;
if (cutlass::sizeof_bits<ElementB>::value == 4) {
scope_max = 2;
scope_min = -2;
} else if (cutlass::sizeof_bits<ElementB>::value == 1) {
scope_max = 2;
scope_min = 0;
}
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(
tensor_B.host_view(), seed + 16, scope_max, scope_min, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_B.host_data(),
tensor_B.capacity());
} else if (init_B == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
cutlass::reference::host::TensorFill(
tensor_C.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_computed.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_reference.host_view(),
ElementC(0)
);
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D_computed.sync_device();
// launch kernel
kernel_transform<Mma, ThreadblockShape><<<dim3(1, 1), dim3(32, 1, 1)>>>(
tensor_D_computed.device_data(), tensor_A.device_data(),
tensor_B.device_data(), tensor_C.device_data());
// verify no errors
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess) << "CUDA ERROR: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
tensor_D_computed.sync_host();
//
// Reference implementation
//
cutlass::reference::host::Gemm<ElementA, LayoutA, ElementB, LayoutB,
ElementC, LayoutC, ElementC, ElementC,
Operator>
reference_gemm;
reference_gemm(
{Shape::kM, Shape::kN, ThreadblockShape::kK},
ElementC(1),
tensor_A.host_ref(),
tensor_B.host_ref(),
ElementC(0),
tensor_D_reference.host_ref()
);
//
// Verify equivalence
//
// compare
bool passed = cutlass::reference::host::TensorEquals(
tensor_D_computed.host_view(),
tensor_D_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() << "):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
<< "Reference:\n" << tensor_D_reference.host_view() << "\n\n"
<< "Computed:\n" << tensor_D_computed.host_view() << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Structure to compute the matrix product
template <
/// Warp-level matrix multiply-accumulate
typename Mma_,
/// Size of threadblock-scoped shape used to store SMEM
typename ThreadblockShape_
>
struct TransformedTestbedComplex {
/// Thread-level matrix multiply-accumulate operator
using Mma = Mma_;
using ThreadblockShape = ThreadblockShape_;
using Shape = typename Mma::Shape;
using ElementA = typename Mma::ElementA;
using LayoutA = typename Mma::LayoutA;
using ElementB = typename Mma::ElementB;
using LayoutB = typename Mma::LayoutB;
using ElementC = typename Mma::ElementC;
using LayoutC = typename Mma::LayoutC;
//
// Data members
//
cutlass::HostTensor<ElementA, LayoutA> tensor_A;
cutlass::HostTensor<ElementB, LayoutB> tensor_B;
cutlass::HostTensor<ElementC, LayoutC> tensor_C;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_computed;
cutlass::HostTensor<ElementC, LayoutC> tensor_D_reference;
//
// Methods
//
/// Allocates workspace in device memory
TransformedTestbedComplex() {
tensor_A.reset(cutlass::make_Coord(ThreadblockShape::kM, ThreadblockShape::kK));
tensor_B.reset(cutlass::make_Coord(ThreadblockShape::kK, ThreadblockShape::kN));
tensor_C.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_computed.reset(cutlass::make_Coord(Shape::kM, Shape::kN));
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
}
/// Runs the test
bool run(
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
//
// initialize device memory
//
if (init_A == cutlass::Distribution::Uniform) {
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(tensor_A.host_view(),
seed, 8, -8, 0);
} else if (init_A == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_A.host_data(),
tensor_A.capacity());
} else if (init_A == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_A.host_view());
} else {
// TODO: Implement the rest
return false;
}
if (init_B == cutlass::Distribution::Uniform) {
uint64_t seed = 7;
cutlass::reference::host::TensorFillRandomUniform(tensor_B.host_view(),
seed + 16, 8, -8, 0);
} else if (init_B == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(tensor_B.host_data(),
tensor_B.capacity());
} else if (init_B == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(tensor_B.host_view());
} else {
// TODO: Implement the rest
return false;
}
cutlass::reference::host::TensorFill(
tensor_C.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_computed.host_view(),
ElementC(0)
);
cutlass::reference::host::TensorFill(
tensor_D_reference.host_view(),
ElementC(0)
);
tensor_A.sync_device();
tensor_B.sync_device();
tensor_C.sync_device();
tensor_D_computed.sync_device();
// launch kernel
kernel_transform<Mma, ThreadblockShape><<< dim3(1, 1), dim3(32, 1, 1) >>>(
tensor_D_computed.device_data(),
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data());
// verify no errors
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess) << "CUDA ERROR: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
tensor_D_computed.sync_host();
//
// Reference implementation
//
cutlass::reference::host::GemmComplex(
{Shape::kM, Shape::kN, ThreadblockShape::kK},
ElementC(1),
tensor_A.host_ref(),
Mma::kTransformA,
tensor_B.host_ref(),
Mma::kTransformB,
ElementC(0),
tensor_C.host_ref(),
tensor_D_reference.host_ref()
);
//
// Verify equivalence
//
// compare
bool passed = cutlass::reference::host::TensorEquals(
tensor_D_computed.host_view(),
tensor_D_reference.host_view()
);
EXPECT_TRUE(passed);
if (!passed) {
cutlass::TensorView<ElementA, cutlass::layout::ColumnMajor> tensor_A_physical(
tensor_A.host_data(),
tensor_A.stride(),
tensor_A.extent());
cutlass::TensorView<ElementB, cutlass::layout::RowMajor> tensor_B_physical(
tensor_B.host_data(),
tensor_B.stride(),
tensor_B.extent());
std::cout <<"cutlass::sizeof_bits<ElementA>::value = "<<cutlass::sizeof_bits<ElementA>::value<<"\n";
std::cout
<< "A:\n" << tensor_A.host_view() << "\n\n"
<< "A(physical - stride: " << tensor_A.stride() << ", extent: " << tensor_A.extent() << "):\n" << tensor_A_physical << "\n\n";
std::cout <<"cutlass::sizeof_bits<ElementB>::value = "<<cutlass::sizeof_bits<ElementB>::value<<"\n";
std::cout
<< "B:\n" << tensor_B.host_view() << "\n\n"
<< "B(physical - stride: " << tensor_B.stride() << ", extent: " << tensor_B.extent() <<"):\n" << tensor_B_physical << "\n\n";
std::cout
<< "C:\n" << tensor_C.host_view() << "\n\n"
<< "Reference:\n" << tensor_D_reference.host_view() << "\n\n"
<< "Computed:\n" << tensor_D_computed.host_view() << std::endl;
}
return passed;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace warp
} // namespace gemm
} // namespace test