Manish Gupta
@@ -34,6 +34,7 @@ add_custom_target(
|
||||
cutlass_test_unit_gemm_device_wmma
|
||||
cutlass_test_unit_gemm_device_tensorop_planar_complex
|
||||
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
|
||||
cutlass_test_unit_gemv_device
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
@@ -50,6 +51,7 @@ add_custom_target(
|
||||
test_unit_gemm_device_wmma
|
||||
test_unit_gemm_device_tensorop_planar_complex
|
||||
test_unit_gemm_device_sparse_tensorop_sm80
|
||||
test_unit_gemv_device
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
@@ -66,6 +68,11 @@ cutlass_test_unit_add_executable(
|
||||
simt_cgemm_tn_sm50.cu
|
||||
simt_cgemm_tt_sm50.cu
|
||||
|
||||
simt_qgemm_nn_sm50.cu
|
||||
simt_qgemm_nt_sm50.cu
|
||||
simt_qgemm_tn_sm50.cu
|
||||
simt_qgemm_tt_sm50.cu
|
||||
|
||||
simt_dgemm_nn_sm50.cu
|
||||
simt_dgemm_nt_sm50.cu
|
||||
simt_dgemm_tn_sm50.cu
|
||||
@@ -203,6 +210,7 @@ cutlass_test_unit_add_executable(
|
||||
|
||||
gemm_f32n_f32n_f32t_tensor_op_f32_sm80.cu
|
||||
gemm_f32n_f32n_f32t_tensor_op_bf16_f32_sm80.cu
|
||||
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
@@ -332,3 +340,36 @@ cutlass_test_unit_add_executable(
|
||||
gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
|
||||
)
|
||||
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemv_device
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
|
||||
gemv.cu
|
||||
)
|
||||
|
||||
if (NOT CUDA_COMPILER MATCHES "[Cc]lang")
|
||||
|
||||
add_dependencies(
|
||||
cutlass_test_unit_gemm_device
|
||||
cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
)
|
||||
|
||||
add_dependencies(
|
||||
test_unit_gemm_device
|
||||
test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
|
||||
gemm_with_reduction_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
gemm_with_broadcast_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
|
||||
gemm_with_reduction_f16t_f16n_f16n_tensorop_f32_sm80.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
|
||||
@@ -41,7 +41,6 @@
|
||||
#include "testbed.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64n_f64t_f64t_tensor_op_f64, 32x32x16_16x16x16) {
|
||||
@@ -209,4 +208,45 @@ TEST(SM80_Device_Gemm_f64n_f64t_f64t_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64an_f64at_f64at_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
|
||||
using ElementOutput = double;
|
||||
using ElementAccumulator = double;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
double,
|
||||
LayoutA,
|
||||
double,
|
||||
LayoutB,
|
||||
ElementOutput,
|
||||
LayoutC,
|
||||
ElementAccumulator,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 16>,
|
||||
cutlass::gemm::GemmShape<32, 64, 16>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
1,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
@@ -209,4 +209,45 @@ TEST(SM80_Device_Gemm_f64t_f64n_f64t_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64at_f64an_f64at_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
|
||||
using ElementOutput = double;
|
||||
using ElementAccumulator = double;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
double,
|
||||
LayoutA,
|
||||
double,
|
||||
LayoutB,
|
||||
ElementOutput,
|
||||
LayoutC,
|
||||
ElementAccumulator,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 16>,
|
||||
cutlass::gemm::GemmShape<32, 64, 16>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
1,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
@@ -126,6 +126,222 @@ TEST(SM70_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_884, 64x64x32_32
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_s884_nn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
float,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
float,
|
||||
4,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_s884_nn : gemm_planar_complex_s884_nn_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f32n_tensor_op_f32_884, 128x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_s884_nn>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s884_f16_nn_128x64_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
cutlass::half_t,
|
||||
8,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s884_f16_nn_128x64_32x2 : gemm_planar_complex_f16_s884_f16_nn_128x64_32x2_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f16n_tensor_op_f32_884, 128x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_nn_128x64_32x2>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s884_f16_nn_64x128_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<64, 128, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
cutlass::half_t,
|
||||
8,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s884_f16_nn_64x128_32x2 : gemm_planar_complex_f16_s884_f16_nn_64x128_32x2_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmPlanarComplex_f16n_f16n_f16n_tensor_op_f32_884, 64x128x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_nn_64x128_32x2>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s884_f16_tt_128x64_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
cutlass::half_t,
|
||||
8,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s884_f16_tt_128x64_32x2 : gemm_planar_complex_f16_s884_f16_tt_128x64_32x2_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmPlanarComplex_f16t_f16t_f16n_tensor_op_f32_884, 128x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_tt_128x64_32x2>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s884_f16_tt_64x128_32x2_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<64, 128, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
cutlass::half_t,
|
||||
8,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s884_f16_tt_64x128_32x2 : gemm_planar_complex_f16_s884_f16_tt_64x128_32x2_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmPlanarComplex_f16t_f16t_f16n_tensor_op_f32_884, 64x128x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s884_f16_tt_64x128_32x2>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM70_SUPPORTED)
|
||||
|
||||
@@ -81,6 +81,48 @@ TEST(SM80_Device_GemmPlanarComplex_f16t_f16n_f32n_tensor_op_f32_16816, 64x64x32_
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s16816_tn_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
float,
|
||||
4,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s16816_tn : gemm_planar_complex_f16_s16816_tn_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmPlanarComplex_f16t_f16n_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_tn>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -127,6 +169,49 @@ TEST(SM80_Device_GemmPlanarComplex_f16h_f16c_f32n_tensor_op_f32_16816, 64x64x32_
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s16816_hc_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kConjugate,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kConjugate,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
float,
|
||||
4,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s16816_hc : gemm_planar_complex_f16_s16816_hc_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmPlanarComplex_f16h_f16c_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_hc>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_s16816_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
@@ -168,6 +253,50 @@ TEST(SM80_Device_GemmPlanarComplex_f16n_f16t_f32n_tensor_op_f32_16816, 64x64x32_
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_f16_s16816_nt_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kNone,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
float,
|
||||
4,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_f16_s16816_nt : gemm_planar_complex_f16_s16816_nt_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmPlanarComplex_f16n_f16t_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_f16_s16816_nt_base>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_s16816_ch_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
@@ -213,4 +342,46 @@ TEST(SM80_Device_GemmPlanarComplex_f16c_f16h_f32n_tensor_op_f32_16816, 64x64x32_
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
using gemm_planar_complex_cf16_s16816_ch_base = typename cutlass::gemm::kernel::DefaultGemmPlanarComplexUniversal<
|
||||
cutlass::half_t,
|
||||
cutlass::layout::ColumnMajor,
|
||||
cutlass::ComplexTransform::kConjugate,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::ComplexTransform::kConjugate,
|
||||
8,
|
||||
cutlass::half_t,
|
||||
cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<32, 32, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
float,
|
||||
4,
|
||||
float,
|
||||
float
|
||||
>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
struct gemm_planar_complex_cf16_s16816_ch : gemm_planar_complex_cf16_s16816_ch_base {
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmPlanarComplex_f16c_f16h_f16n_tensor_op_f32_16816, 64x64x32_32x32x32) {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<gemm_planar_complex_cf16_s16816_ch>;
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemmPlanarComplex<Gemm>());
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/functional.h"
|
||||
|
||||
#include "cutlass/gemm/kernel/default_gemm_with_broadcast.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination_bias_relu.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed_gemm_with_broadcast.h"
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes:
|
||||
///
|
||||
/// Z = GEMM+Bias+ReLu
|
||||
/// T = Relu conditional
|
||||
///
|
||||
template <typename Gemm>
|
||||
struct GemmWithBiasReluReferenceOp {
|
||||
|
||||
using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
|
||||
|
||||
using ElementCompute = typename OutputOp::ElementCompute;
|
||||
using ElementZ = typename OutputOp::ElementZ;
|
||||
using ElementT = typename OutputOp::ElementT;
|
||||
|
||||
typename OutputOp::BinaryOp binary_op;
|
||||
typename OutputOp::ElementwiseOp elementwise_op;
|
||||
|
||||
GemmWithBiasReluReferenceOp() { }
|
||||
|
||||
void operator()(ElementZ &Z, ElementT &T, ElementCompute gemm, ElementCompute bias) {
|
||||
|
||||
ElementCompute kThreshold = ElementCompute();
|
||||
|
||||
ElementCompute z_full = binary_op(gemm, bias);
|
||||
|
||||
bool conditional = (z_full >= kThreshold);
|
||||
|
||||
if (!conditional) {
|
||||
z_full = kThreshold;
|
||||
}
|
||||
|
||||
Z = ElementZ(z_full);
|
||||
T = ElementT(conditional);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
cutlass::epilogue::thread::GELU_taylor<float>
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 8>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
cutlass::epilogue::thread::GELU_taylor<float>
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
true
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 8>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
true
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // if defiend(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x5_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
cutlass::epilogue::thread::GELU_taylor<float>
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
5,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
|
||||
}
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x5_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
true
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
5,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm>>();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x4_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
cutlass::epilogue::thread::GELU_taylor<float>
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
4,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
|
||||
}
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x4_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
true
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
4,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm>>();
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_GELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x3_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasElementwise<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
cutlass::epilogue::thread::GELU_taylor<float>
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm>();
|
||||
}
|
||||
|
||||
TEST(SM80_Device_GemmWithBroadcast_RELU_f16n_f16n_f16n_tensor_op_f32, 128x128_32x3_64x64x32_16x8x16) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationBiasRelu<
|
||||
cutlass::half_t,
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8,
|
||||
true
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithBroadcast<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
3,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
test::gemm::device::TestAllGemmWithBroadcast<Gemm, GemmWithBiasReluReferenceOp<Gemm> >();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,378 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/functional.h"
|
||||
|
||||
#include "cutlass/gemm/kernel/default_gemm_with_reduction.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_drelu.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination_dgelu.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed_gemm_with_reduction.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct dReluLambda {
|
||||
float operator()(float d_y, float t) {
|
||||
if (t <= 0) {
|
||||
d_y = 0;
|
||||
}
|
||||
return d_y;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 8>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::GemmWithReductionReference<
|
||||
Gemm,
|
||||
dReluLambda
|
||||
>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
2,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 256x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
cutlass::gemm::GemmShape<256, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 8>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::GemmWithReductionReference<
|
||||
Gemm,
|
||||
dReluLambda
|
||||
>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
1,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::GemmWithReductionReference<
|
||||
Gemm,
|
||||
dReluLambda
|
||||
>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
2,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmWithReduction_dReLU_bGrad_f16n_f16n_f16n_tensor_op_f32, 256x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<256, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::GemmWithReductionReference<
|
||||
Gemm,
|
||||
dReluLambda
|
||||
>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
1,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
|
||||
template <typename Gemm>
|
||||
struct Gemm_dReLU_packed_bits_reference_op {
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::ElementCompute;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Gemm_dReLU_packed_bits_reference_op() { }
|
||||
|
||||
ElementCompute operator()(
|
||||
ElementAccumulator d_y,
|
||||
ElementT t) const {
|
||||
|
||||
ElementCompute result = ElementCompute(d_y);
|
||||
|
||||
bool cond = bool(t);
|
||||
if (!cond) {
|
||||
result = ElementCompute();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace device
|
||||
} // namespace gemm
|
||||
} // namespace test
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Device_GemmWithReduction_dReLU_conditional_bits_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDReluConditionalBits<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 8>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::Gemm_dReLU_packed_bits_reference_op<Gemm>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
2,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM70_Device_GemmWithReduction_dReLU_conditional_bits_bGrad_f16n_f16n_f16n_tensor_op_f32, 128x128x32_64x64x8) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDReluConditionalBits<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<8, 8, 4>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
2,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::Gemm_dReLU_packed_bits_reference_op<Gemm>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{520, 264, 96},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
2,
|
||||
float(1.25),
|
||||
float(2.25)
|
||||
);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // if defiend(CUTLASS_ARCH_MMA_SM75_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,112 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/functional.h"
|
||||
|
||||
#include "cutlass/gemm/kernel/default_gemm_with_reduction.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_drelu.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination_dgelu.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed_gemm_with_reduction.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct dReluLambda {
|
||||
float operator()(float d_y, float t) {
|
||||
if (t <= 0) {
|
||||
d_y = 0;
|
||||
}
|
||||
return d_y;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_GemmWithReduction_dReLU_bGrad_f16t_f16n_f16n_tensor_op_f32, 128x128x32_64x64x32) {
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationDRelu<
|
||||
float,
|
||||
float,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
8
|
||||
>;
|
||||
|
||||
using GemmKernel =
|
||||
typename cutlass::gemm::kernel::DefaultGemmWithReduction<
|
||||
cutlass::half_t, cutlass::layout::RowMajor, cutlass::ComplexTransform::kNone, 8, // transposed B operand
|
||||
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::ComplexTransform::kNone, 8, // transposed A operand
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 32>,
|
||||
cutlass::gemm::GemmShape<64, 64, 32>,
|
||||
cutlass::gemm::GemmShape<16, 8, 16>,
|
||||
EpilogueOutputOp,
|
||||
cutlass::plus<float>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>,
|
||||
5,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>::GemmKernel;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using ReferenceOp = test::gemm::device::GemmWithReductionReference<
|
||||
Gemm,
|
||||
dReluLambda
|
||||
>;
|
||||
|
||||
test::gemm::device::TestGemmWithReduction<Gemm, ReferenceOp>(
|
||||
{8, 8, 136},
|
||||
cutlass::gemm::GemmUniversalMode::kGemm
|
||||
);
|
||||
}
|
||||
|
||||
#endif // if defiend(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,438 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMV interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/kernel/gemv.h"
|
||||
#include "cutlass/gemm/device/gemv.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/distribution.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/tensor_norm.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
#include "cutlass/util/reference/host/gemm_complex.h"
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
|
||||
template <typename Gemv>
|
||||
class TestbedGemv {
|
||||
public:
|
||||
|
||||
using ElementA = typename Gemv::ElementA;
|
||||
using LayoutA = typename Gemv::LayoutA;
|
||||
using ElementB = typename Gemv::ElementB;
|
||||
using ElementC = typename Gemv::ElementC;
|
||||
|
||||
using ElementAccumulator = typename Gemv::ElementAccumulator;
|
||||
using ElementCompute = typename Gemv::EpilogueOutputOp::ElementCompute;
|
||||
|
||||
using LayoutV = cutlass::layout::RowMajor;
|
||||
|
||||
private:
|
||||
|
||||
/// Initialization
|
||||
cutlass::Distribution::Kind init_A;
|
||||
cutlass::Distribution::Kind init_B;
|
||||
cutlass::Distribution::Kind init_C;
|
||||
uint64_t seed;
|
||||
|
||||
cutlass::HostTensor<ElementA, LayoutA> tensor_A;
|
||||
cutlass::HostTensor<ElementB, LayoutV> tensor_B;
|
||||
cutlass::HostTensor<ElementC, LayoutV> tensor_C;
|
||||
cutlass::HostTensor<ElementC, LayoutV> tensor_D;
|
||||
cutlass::HostTensor<ElementC, LayoutV> reference_D;
|
||||
|
||||
public:
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TestbedGemv(
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = 2080
|
||||
):
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
/// Helper to initialize a tensor view
|
||||
template <typename Element, typename Layout>
|
||||
bool initialize_tensor(
|
||||
cutlass::TensorView<Element, Layout> view,
|
||||
cutlass::Distribution::Kind dist_kind,
|
||||
uint64_t seed) {
|
||||
|
||||
if (dist_kind == cutlass::Distribution::Uniform) {
|
||||
|
||||
double scope_max, scope_min;
|
||||
int bits_input = cutlass::sizeof_bits<Element>::value;
|
||||
int bits_output = cutlass::sizeof_bits<typename Gemv::ElementC>::value;
|
||||
|
||||
if (bits_input == 1) {
|
||||
scope_max = 2;
|
||||
scope_min = 0;
|
||||
} else if (bits_input <= 8) {
|
||||
scope_max = 2;
|
||||
scope_min = -2;
|
||||
} else if (bits_output == 16) {
|
||||
scope_max = 5;
|
||||
scope_min = -5;
|
||||
} else {
|
||||
scope_max = 8;
|
||||
scope_min = -8;
|
||||
}
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
view, seed, scope_max, scope_min, 0);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Identity) {
|
||||
|
||||
cutlass::reference::host::TensorFillIdentity(view);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Gaussian) {
|
||||
|
||||
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Sequential) {
|
||||
|
||||
cutlass::reference::host::BlockFillSequential(
|
||||
view.data(), view.capacity());
|
||||
}
|
||||
else {
|
||||
// TODO: Implement the rest
|
||||
EXPECT_TRUE(false) << "Not implemented";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Initializes data structures
|
||||
void initialize(
|
||||
cutlass::MatrixCoord problem_size
|
||||
) {
|
||||
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
|
||||
tensor_A.resize(problem_size);
|
||||
tensor_B.resize({problem_size.column(), 1});
|
||||
tensor_C.resize({problem_size.row(), 1});
|
||||
tensor_D.resize({problem_size.row(), 1});
|
||||
reference_D.resize({problem_size.row(), 1}, false);
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
|
||||
|
||||
// It is possible to randomly initialize to all zeros, so override this with non-zeros
|
||||
// in the upper left corner of each operand.
|
||||
tensor_A.host_view().at({0, 0}) = typename Gemv::ElementA(1);
|
||||
tensor_B.host_view().at({0, 0}) = typename Gemv::ElementB(1);
|
||||
tensor_C.host_view().at({0, 0}) = typename Gemv::ElementC(1);
|
||||
|
||||
cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
|
||||
|
||||
tensor_A.sync_device();
|
||||
tensor_B.sync_device();
|
||||
tensor_C.sync_device();
|
||||
tensor_D.sync_device();
|
||||
}
|
||||
|
||||
/// Compares computed reference with device reference and outputs to a file if incorrect
|
||||
bool compare_reference(
|
||||
cutlass::MatrixCoord problem_size,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
tensor_D.sync_host();
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
|
||||
|
||||
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
|
||||
|
||||
EXPECT_TRUE(passed) << " mismatched reference";
|
||||
|
||||
if (!passed) {
|
||||
|
||||
std::ofstream file("testbed_universal_errors.txt");
|
||||
|
||||
file
|
||||
<< "problem: " << problem_size
|
||||
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
|
||||
|
||||
file
|
||||
<< "A =\n" << tensor_A.host_view()
|
||||
<< "\nB =\n" << tensor_B.host_view()
|
||||
<< "\nC =\n" << tensor_C.host_view()
|
||||
<< "\n\nReference =\n" << reference_D.host_view()
|
||||
<< "\nComputed =\n" << tensor_D.host_view();
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/// Verifies the result is a GEMM
|
||||
bool verify(
|
||||
cutlass::MatrixCoord problem_size,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
cutlass::reference::host::GemmComplex<
|
||||
typename Gemv::ElementA, typename Gemv::LayoutA,
|
||||
typename Gemv::ElementB, LayoutV,
|
||||
typename Gemv::ElementC, LayoutV,
|
||||
ElementCompute, ElementAccumulator
|
||||
>(
|
||||
{problem_size.row(), 1, problem_size.column()},
|
||||
alpha,
|
||||
tensor_A.host_ref(),
|
||||
Gemv::kTransformA,
|
||||
tensor_B.host_ref(),
|
||||
Gemv::kTransformB,
|
||||
beta,
|
||||
tensor_C.host_ref(),
|
||||
reference_D.host_ref(),
|
||||
ElementAccumulator(0)
|
||||
);
|
||||
|
||||
return compare_reference(problem_size, alpha, beta);
|
||||
}
|
||||
|
||||
/// Runs one problem size
|
||||
bool run(
|
||||
cutlass::MatrixCoord problem_size,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
this->initialize(problem_size);
|
||||
|
||||
//
|
||||
// Initialize the GEMM operator
|
||||
//
|
||||
|
||||
typename Gemv::Arguments arguments{
|
||||
problem_size,
|
||||
{alpha, beta},
|
||||
tensor_A.device_ref(),
|
||||
tensor_B.device_data(),
|
||||
tensor_C.device_data(),
|
||||
tensor_D.device_data(),
|
||||
tensor_B.layout().stride(0),
|
||||
tensor_C.layout().stride(0),
|
||||
tensor_D.layout().stride(0)
|
||||
};
|
||||
|
||||
Gemv gemm_op;
|
||||
|
||||
size_t workspace_size = Gemv::get_workspace_size(arguments);
|
||||
|
||||
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
|
||||
|
||||
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
|
||||
status = gemm_op();
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
bool passed = this->verify(problem_size, alpha, beta);
|
||||
|
||||
return passed;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemv>
|
||||
bool TestAllGemv() {
|
||||
|
||||
using ElementCompute = typename Gemv::EpilogueOutputOp::ElementCompute;
|
||||
|
||||
int M[] = {
|
||||
8, 48, 192, 520
|
||||
};
|
||||
|
||||
int K[] = {
|
||||
8, 192, 528
|
||||
};
|
||||
|
||||
double Alpha[] = {
|
||||
1, 1.25
|
||||
};
|
||||
|
||||
double Beta[] = {
|
||||
0, 1, 1.25
|
||||
};
|
||||
|
||||
for (int m : M) {
|
||||
for (int k : K) {
|
||||
for (double alpha : Alpha) {
|
||||
for (double beta : Beta) {
|
||||
|
||||
TestbedGemv<Gemv> testbed;
|
||||
|
||||
if (!testbed.run({m, k}, ElementCompute(alpha), ElementCompute(beta))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace gemm
|
||||
} // namespace test
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemv_f32n_f32_f32_simt_f32, Simple) {
|
||||
|
||||
using ElementOutput = float;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementAccumulator = float;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
1,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator>;
|
||||
|
||||
using Gemv = cutlass::gemm::device::Gemv<
|
||||
cutlass::gemm::kernel::Gemv<
|
||||
ElementOutput, // Element A
|
||||
LayoutA, // Layout A
|
||||
ElementOutput, // Element B
|
||||
ElementOutput, // Element C
|
||||
ElementAccumulator, // Element Accumulator
|
||||
EpilogueOp // Output operator
|
||||
>
|
||||
>;
|
||||
|
||||
|
||||
EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemv_f16n_f16_f32_simt_f32, Simple) {
|
||||
|
||||
using ElementInput = cutlass::half_t;
|
||||
using ElementOutput = float;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementAccumulator = float;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
1,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator>;
|
||||
|
||||
using Gemv = cutlass::gemm::device::Gemv<
|
||||
cutlass::gemm::kernel::Gemv<
|
||||
ElementInput, // Element A
|
||||
LayoutA, // Layout A
|
||||
ElementInput, // Element B
|
||||
ElementOutput, // Element C
|
||||
ElementAccumulator, // Element Accumulator
|
||||
EpilogueOp // Output operator
|
||||
>
|
||||
>;
|
||||
|
||||
|
||||
EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemv_f16n_f16_f16_simt_f32, Simple) {
|
||||
|
||||
using ElementInput = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementAccumulator = float;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
1,
|
||||
ElementAccumulator,
|
||||
ElementAccumulator>;
|
||||
|
||||
using Gemv = cutlass::gemm::device::Gemv<
|
||||
cutlass::gemm::kernel::Gemv<
|
||||
ElementInput, // Element A
|
||||
LayoutA, // Layout A
|
||||
ElementInput, // Element B
|
||||
ElementOutput, // Element C
|
||||
ElementAccumulator, // Element Accumulator
|
||||
EpilogueOp // Output operator
|
||||
>
|
||||
>;
|
||||
|
||||
|
||||
EXPECT_TRUE(test::gemm::TestAllGemv<Gemv>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 256x32x8_64x16x1_8x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_nn, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_nn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 256x32x8_64x16x1_8x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tn, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -673,66 +673,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1093,96 +1033,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 256x32x8_64x16x1_8x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1243,36 +1093,6 @@ CUTLASS_TEST_L1(SM50_device_cgemm_tt, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_cgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = cutlass::complex<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1302,4 +1122,3 @@ CUTLASS_TEST_L2(SM50_device_cgemm_tt, 128x64x8_32x16x1_4x4_8x4_4x4, {
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
|
||||
@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_nn, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_dgemm_affin2_nn, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_nn, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_nn, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_nn, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_nn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
|
||||
@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_nt, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_dgemm_affine2_nt, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -643,6 +643,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_tn, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_dgemm_affine2_tn, 64x64x8_32x32x1_8x4_4x8_2x2, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_tn, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tn, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tn, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
|
||||
@@ -163,6 +163,45 @@ CUTLASS_TEST_L0(SM50_device_dgemm_tt, 32x32x8_32x32x1_8x4_4x8_1x1, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_dgemm_affine2_tt, 32x32x8_32x32x1_8x4_4x8_1x1, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -673,66 +712,6 @@ CUTLASS_TEST_L1(SM50_device_dgemm_tt, 128x32x8_64x16x1_8x4_8x4_2x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -973,96 +952,6 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tt, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1094,32 +983,3 @@ CUTLASS_TEST_L2(SM50_device_dgemm_tt, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_dgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = double;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
|
||||
@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1483,36 +1453,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1483,36 +1453,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1633,36 +1573,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tn, 64x256x8_16x64x1_4x8_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -1633,36 +1633,6 @@ CUTLASS_TEST_L2(SM50_device_igemm_tt, 64x256x8_16x64x1_4x8_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_igemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = int;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -0,0 +1,855 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/device/gemm.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 8x32x8_8x32x1_2x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_nn, 16x32x8_16x32x1_4x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 8x32x8_8x16x1_2x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 8x64x8_8x32x1_2x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 16x32x8_16x16x1_4x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 16x64x8_16x32x1_4x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x32x8_32x16x1_4x4_8x4_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 1
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x32x1_4x4_4x8_2x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x32x8_8x16x1_2x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x64x8_8x32x1_2x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x16x1_4x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_nn, 32x64x8_16x32x1_4x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x32x8_32x16x1_4x4_8x4_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_16x8x1_2x2_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x64x8_16x16x1_4x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 32x128x8_16x32x1_4x4_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x64x8_32x16x1_4x4_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x32x8_8x16x1_2x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x32x8_16x16x1_4x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x64x8_16x32x1_4x4_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 128 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nn, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nn, 128x64x8_32x16x1_4x4_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
@@ -0,0 +1,855 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/device/gemm.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 8x32x8_8x32x1_2x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_nt, 16x32x8_16x32x1_4x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 8x32x8_8x16x1_2x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 8x64x8_8x32x1_2x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 16x32x8_16x16x1_4x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 16x64x8_16x32x1_4x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x32x8_32x16x1_4x4_8x4_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 1
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x32x1_4x4_4x8_2x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x32x8_8x16x1_2x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x64x8_8x32x1_2x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x16x1_4x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_nt, 32x64x8_16x32x1_4x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x32x8_32x16x1_4x4_8x4_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_16x8x1_2x2_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x64x8_16x16x1_4x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 32x128x8_16x32x1_4x4_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x64x8_32x16x1_4x4_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x32x8_8x16x1_2x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x32x8_16x16x1_4x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x64x8_16x32x1_4x4_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 128 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_nt, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_nt, 128x64x8_32x16x1_4x4_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
@@ -0,0 +1,855 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/device/gemm.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 8x32x8_8x32x1_2x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_tn, 16x32x8_16x32x1_4x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 8x32x8_8x16x1_2x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 8x64x8_8x32x1_2x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 16x32x8_16x16x1_4x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 16x64x8_16x32x1_4x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x32x8_32x16x1_4x4_8x4_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 1
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x32x1_4x4_4x8_2x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x32x8_8x16x1_2x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x64x8_8x32x1_2x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x16x1_4x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_tn, 32x64x8_16x32x1_4x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x32x8_32x16x1_4x4_8x4_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_16x8x1_2x2_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x64x8_16x16x1_4x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 32x128x8_16x32x1_4x4_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x64x8_32x16x1_4x4_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x32x8_8x16x1_2x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x32x8_16x16x1_4x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x64x8_16x32x1_4x4_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 128 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tn, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tn, 128x64x8_32x16x1_4x4_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
@@ -0,0 +1,855 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm/device/gemm.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 8x32x8_8x32x1_2x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 1
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_tt, 16x32x8_16x32x1_4x4_4x8_1x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 8x32x8_8x16x1_2x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 8 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 8x64x8_8x32x1_2x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<8, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 16x32x8_16x16x1_4x2_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 16x64x8_16x32x1_4x4_4x8_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 1 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x32x8_32x16x1_4x4_8x4_1x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 1
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x32x1_4x4_4x8_2x1, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x32x8_8x16x1_2x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 16 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x64x8_8x32x1_2x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x16x1_4x2_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_qgemm_tt, 32x64x8_16x32x1_4x4_4x8_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x32x8_32x16x1_4x4_8x4_2x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_16x8x1_2x2_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x64x8_16x16x1_4x2_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 32 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 32x128x8_16x32x1_4x4_4x8_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x64x8_32x16x1_4x4_8x4_2x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 32 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x32x8_8x16x1_2x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 32 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x32x8_16x16x1_4x2_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x64x8_16x32x1_4x4_4x8_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 2
|
||||
// Threadblock: 128 x 32 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 128x32x8_32x16x1_4x4_8x4_4x2, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 64 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 16, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x32x16_16x8x1_2x2_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 64 x 8
|
||||
CUTLASS_TEST_L2(SM50_device_qgemm_tt, 64x64x8_16x16x1_4x2_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 64 x 128 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 64x128x8_16x32x1_4x4_4x8_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<64, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 64 x 8
|
||||
CUTLASS_TEST_L1(SM50_device_qgemm_tt, 128x64x8_32x16x1_4x4_8x4_4x4, {
|
||||
using precision = cutlass::Quaternion<float>;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 16, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_nn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_nn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1213,6 +1183,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_nn, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 8
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 128 x 128 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_sgemm_affine2_nn, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1483,36 +1492,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_nn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_nn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -1213,6 +1213,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_nt, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 8
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 128 x 128 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_sgemm_affine2_nt, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
|
||||
@@ -130,6 +130,43 @@ TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 128x128x8_32x64x1) {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM80_Device_Gemm_f32an_f32at_f32at_simt_f32, 128x128x8_32x64x1) {
|
||||
|
||||
using Element = float;
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
Element,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 8>,
|
||||
cutlass::gemm::GemmShape<32, 64, 8>,
|
||||
cutlass::gemm::GemmShape<1, 1, 1>,
|
||||
cutlass::epilogue::thread::LinearCombination<
|
||||
Element,
|
||||
1,
|
||||
Element,
|
||||
Element>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C ));
|
||||
|
||||
}
|
||||
|
||||
TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 64x128x8_32x64x1) {
|
||||
|
||||
using Element = float;
|
||||
@@ -248,7 +285,6 @@ TEST(SM80_Device_Gemm_f32n_f32t_f32t_simt_f32, 128x256x8_64x64x1) {
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -943,36 +943,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 16x64x16_8x16x1_2x2_4x8_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 16 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_tn, 16x128x16_8x32x1_2x4_4x8_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<16, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1213,6 +1183,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_tn, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 8
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 128 x 128 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_sgemm_affine2_tn, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1483,36 +1492,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 32x64x16_8x16x1_2x2_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 4
|
||||
// Threads / Warp: 4 x 8
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 32 x 128 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_tn, 32x128x16_8x32x1_2x4_4x8_4x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<32, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<8, 32, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
@@ -1633,36 +1612,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tn, 64x256x8_16x64x1_4x8_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_tn, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::ColumnMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -44,7 +44,6 @@
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 32x64x8_32x64x1) {
|
||||
|
||||
using Element = float;
|
||||
@@ -132,6 +131,42 @@ TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 128x128x8_32x64x1) {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM80_Device_Gemm_f32at_f32an_f32t_simt_f32, 128x128x8_32x64x1) {
|
||||
|
||||
using Element = float;
|
||||
using LayoutA = cutlass::layout::AffineRank2RowMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
Element,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm80,
|
||||
cutlass::gemm::GemmShape<128, 128, 8>,
|
||||
cutlass::gemm::GemmShape<32, 64, 8>,
|
||||
cutlass::gemm::GemmShape<1, 1, 1>,
|
||||
cutlass::epilogue::thread::LinearCombination<
|
||||
Element,
|
||||
1,
|
||||
Element,
|
||||
Element>,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
3
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {1};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>( stride_factor_A, stride_factor_B, stride_factor_C ));
|
||||
}
|
||||
|
||||
TEST(SM80_Device_Gemm_f32t_f32n_f32t_simt_f32, 64x128x8_32x64x1) {
|
||||
|
||||
using Element = float;
|
||||
|
||||
@@ -1213,6 +1213,45 @@ CUTLASS_TEST_L0(SM50_device_sgemm_tt, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 8 x 8
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 2 x 4
|
||||
// Threadblock: 128 x 128 x 8
|
||||
CUTLASS_TEST_L0(SM50_device_sgemm_affine2_tt, 128x128x8_64x32x1_8x8_8x4_2x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 8>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using LayoutA = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutB = cutlass::layout::AffineRank2ColumnMajor;
|
||||
using LayoutC = cutlass::layout::AffineRankN<2>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, LayoutA,
|
||||
precision, LayoutB,
|
||||
precision, LayoutC,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
|
||||
typename LayoutA::Stride::Index stride_factor_A[] = {3, 4};
|
||||
typename LayoutB::Stride::Index stride_factor_B[] = {5, 6};
|
||||
typename LayoutC::Stride::Index stride_factor_C[] = {7, 8};
|
||||
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>(stride_factor_A, stride_factor_B, stride_factor_C));
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 2 x 2
|
||||
// Threads / Warp: 4 x 8
|
||||
@@ -1633,36 +1672,6 @@ CUTLASS_TEST_L2(SM50_device_sgemm_tt, 64x256x8_16x64x1_4x8_4x8_4x4, {
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 2
|
||||
// Threads / Warp: 8 x 4
|
||||
// Warps / Block: 4 x 4
|
||||
// Threadblock: 128 x 32 x 16
|
||||
CUTLASS_TEST_L2(SM50_device_sgemm_tt, 128x32x16_32x8x1_4x2_8x4_4x4, {
|
||||
using precision = float;
|
||||
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 8, 16>;
|
||||
|
||||
static int const kEpilogueElementsPerAccess = 1;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
precision, kEpilogueElementsPerAccess, precision, precision>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::Gemm<
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision, cutlass::layout::RowMajor,
|
||||
precision,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
ThreadblockShape, WarpShape, InstructionShape,
|
||||
EpilogueOutputOp,
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
|
||||
2 // Stages
|
||||
>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
|
||||
} )
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Elements / Thread: 4 x 4
|
||||
// Threads / Warp: 8 x 4
|
||||
|
||||
@@ -45,14 +45,15 @@ warpShapeMin = 8*8
|
||||
|
||||
threadblockEdgeMax = 256
|
||||
|
||||
# char, type bits/elem, max tile, L0 threadblock tiles
|
||||
# char, type bits/elem, max tile, L0 threadblock tiles
|
||||
precisions = [
|
||||
["c", "cutlass::complex<float>", 64, 64*128, [ [ 64, 128], [ 64, 32] ] ],
|
||||
["d", "double", 64, 64*64, [ [ 64, 64], [ 32, 32] ] ],
|
||||
["h", "cutlass::half_t", 16, 128*256, [ [256, 128], [ 64, 128], [ 64, 32] ] ],
|
||||
["i", "int", 32, 128*128, [ [128, 64], [ 16, 32] ] ],
|
||||
["s", "float", 32, 128*128, [ [128, 256], [128, 128], [ 64, 64] ] ],
|
||||
["z", "cutlass::complex<double>", 128, 64*64, [ [ 32, 64], [ 16, 32] ] ],
|
||||
["c", "cutlass::complex<float>", 64, 64*128, [ [ 64, 128], [ 64, 32] ] ],
|
||||
["q", "cutlass::Quaternion<float>", 64, 64*128, [ [ 64, 128], [ 64, 32] ] ],
|
||||
["d", "double", 64, 64*64, [ [ 64, 64], [ 32, 32] ] ],
|
||||
["h", "cutlass::half_t", 16, 128*256, [ [256, 128], [ 64, 128], [ 64, 32] ] ],
|
||||
["i", "int", 32, 128*128, [ [128, 64], [ 16, 32] ] ],
|
||||
["s", "float", 32, 128*128, [ [128, 256], [128, 128], [ 64, 64] ] ],
|
||||
["z", "cutlass::complex<double>", 128, 64*64, [ [ 32, 64], [ 16, 32] ] ],
|
||||
]
|
||||
# L1 will have a single kernel for every unique shape
|
||||
# L2 will have everything else
|
||||
@@ -313,7 +314,7 @@ for precision in precisions:
|
||||
" cutlass::arch::Sm50,\n"
|
||||
" ThreadblockShape, WarpShape, InstructionShape,\n"
|
||||
" EpilogueOutputOp,\n"
|
||||
" cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle,\n"
|
||||
" cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,\n"
|
||||
" 2 // Stages\n"
|
||||
" >;\n" % (
|
||||
"Column" if columnMajorA else "Row",
|
||||
|
||||
@@ -45,6 +45,8 @@
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
@@ -58,6 +60,9 @@ struct Testbed {
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
/// Initialization
|
||||
typename Gemm::LayoutA::Stride stride_factor_A;
|
||||
typename Gemm::LayoutB::Stride stride_factor_B;
|
||||
typename Gemm::LayoutC::Stride stride_factor_C;
|
||||
cutlass::Distribution::Kind init_A;
|
||||
cutlass::Distribution::Kind init_B;
|
||||
cutlass::Distribution::Kind init_C;
|
||||
@@ -79,6 +84,23 @@ struct Testbed {
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = 2080
|
||||
):
|
||||
stride_factor_A(typename Gemm::LayoutA::Stride()),
|
||||
stride_factor_B(typename Gemm::LayoutB::Stride()),
|
||||
stride_factor_C(typename Gemm::LayoutC::Stride()),
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
Testbed(
|
||||
typename Gemm::LayoutA::Stride stride_factor_A_,
|
||||
typename Gemm::LayoutB::Stride stride_factor_B_,
|
||||
typename Gemm::LayoutC::Stride stride_factor_C_,
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = 2080
|
||||
):
|
||||
stride_factor_A(stride_factor_A_),
|
||||
stride_factor_B(stride_factor_B_),
|
||||
stride_factor_C(stride_factor_C_),
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
/// Helper to initialize a tensor view
|
||||
@@ -139,11 +161,11 @@ struct Testbed {
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
|
||||
tensor_A.resize(problem_size.mk());
|
||||
tensor_B.resize(problem_size.kn());
|
||||
tensor_C.resize(problem_size.mn());
|
||||
tensor_D.resize(problem_size.mn());
|
||||
reference_D.resize(problem_size.mn(), false);
|
||||
tensor_A.resize(problem_size.mk(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutA>::layout_factory(problem_size.mk(), stride_factor_A));
|
||||
tensor_B.resize(problem_size.kn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutB>::layout_factory(problem_size.kn(), stride_factor_B));
|
||||
tensor_C.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C));
|
||||
tensor_D.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C));
|
||||
reference_D.resize(problem_size.mn(), cutlass::layout::Affine2Layout_Factory<typename Gemm::LayoutC>::layout_factory(problem_size.mn(), stride_factor_C), false);
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
|
||||
@@ -153,7 +175,7 @@ struct Testbed {
|
||||
// in the upper left corner of each operand.
|
||||
tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
|
||||
tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
|
||||
tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
|
||||
tensor_C.host_view().at(cutlass::make_Coord(0, 0)) = typename Gemm::ElementC(1);
|
||||
|
||||
cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
|
||||
|
||||
@@ -226,7 +248,7 @@ struct Testbed {
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
|
||||
cutlass::reference::host::Gemm<
|
||||
typename Gemm::ElementA, typename Gemm::LayoutA,
|
||||
typename Gemm::ElementB, typename Gemm::LayoutB,
|
||||
@@ -347,7 +369,10 @@ struct Testbed {
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemm>
|
||||
bool TestAllGemm() {
|
||||
bool TestAllGemm(
|
||||
const typename Gemm::LayoutA::Stride& stride_factor_A = typename Gemm::LayoutA::Stride(),
|
||||
const typename Gemm::LayoutB::Stride& stride_factor_B = typename Gemm::LayoutB::Stride(),
|
||||
const typename Gemm::LayoutC::Stride& stride_factor_C = typename Gemm::LayoutC::Stride()) {
|
||||
bool passed = true;
|
||||
|
||||
int const kMinimumOperandElementSize =
|
||||
@@ -393,7 +418,7 @@ bool TestAllGemm() {
|
||||
2.0
|
||||
};
|
||||
|
||||
Testbed<Gemm> testbed;
|
||||
Testbed<Gemm> testbed(stride_factor_A, stride_factor_B, stride_factor_C);
|
||||
|
||||
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
|
||||
|
||||
@@ -414,7 +439,6 @@ bool TestAllGemm() {
|
||||
for (auto beta : problem_beta) {
|
||||
|
||||
cutlass::gemm::GemmCoord problem_size(m, n, k);
|
||||
|
||||
passed = testbed.run(
|
||||
problem_size,
|
||||
split_k,
|
||||
|
||||
@@ -0,0 +1,651 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/distribution.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/tensor_norm.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
#include "cutlass/util/reference/host/gemm_complex.h"
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemm>
|
||||
struct GemmWithBroadcastReferenceOp {
|
||||
|
||||
using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
|
||||
|
||||
using ElementCompute = typename OutputOp::ElementCompute;
|
||||
using ElementZ = typename OutputOp::ElementZ;
|
||||
using ElementT = typename OutputOp::ElementT;
|
||||
|
||||
typename OutputOp::BinaryOp binary_op;
|
||||
typename OutputOp::ElementwiseOp elementwise_op;
|
||||
|
||||
GemmWithBroadcastReferenceOp() { }
|
||||
|
||||
void operator()(ElementZ &Z, ElementT &T, ElementCompute gemm, ElementCompute bias) {
|
||||
|
||||
ElementCompute z_full = binary_op(gemm, bias);
|
||||
Z = ElementZ(z_full);
|
||||
|
||||
ElementCompute t_full = elementwise_op(z_full);
|
||||
T = ElementT(t_full);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Fused testbed
|
||||
//
|
||||
// Y = GEMM(AB, C)
|
||||
//
|
||||
// Z[i, j] = ReductionOp(Y[i, j], Broadcast[i])
|
||||
//
|
||||
// T[i, j] = Elementwise(Z[i, j])
|
||||
//
|
||||
|
||||
template <
|
||||
typename Gemm,
|
||||
typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
|
||||
>
|
||||
struct TestbedGemmWithBroadcast {
|
||||
|
||||
using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCOmpute = typename OutputOp::ElementCompute;
|
||||
using ElementZ = typename OutputOp::ElementZ;
|
||||
using ElementT = typename OutputOp::ElementT;
|
||||
|
||||
|
||||
/// Initialization
|
||||
cutlass::Distribution::Kind init_A;
|
||||
cutlass::Distribution::Kind init_B;
|
||||
cutlass::Distribution::Kind init_C;
|
||||
uint64_t seed;
|
||||
|
||||
cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A; // Input A
|
||||
cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B; // Input B
|
||||
cutlass::HostTensor<ElementC, typename Gemm::LayoutC> tensor_C; // Input C
|
||||
cutlass::HostTensor<ElementC, typename Gemm::LayoutC> tensor_Broadcast; // Input Broadcast
|
||||
|
||||
cutlass::HostTensor<ElementZ, typename Gemm::LayoutC> tensor_Z;
|
||||
cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_T;
|
||||
|
||||
cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_C_ref;
|
||||
cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_Y_ref;
|
||||
cutlass::HostTensor<ElementZ, typename Gemm::LayoutC> tensor_Z_ref;
|
||||
cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_T_ref;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TestbedGemmWithBroadcast(
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = 2080
|
||||
):
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
/// Helper to initialize a tensor view
|
||||
template <typename Element, typename Layout>
|
||||
bool initialize_tensor(
|
||||
cutlass::TensorView<Element, Layout> view,
|
||||
cutlass::Distribution::Kind dist_kind,
|
||||
uint64_t seed) {
|
||||
|
||||
if (dist_kind == cutlass::Distribution::Uniform) {
|
||||
|
||||
double scope_max, scope_min;
|
||||
int bits_input = cutlass::sizeof_bits<Element>::value;
|
||||
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
|
||||
|
||||
if (bits_input == 1) {
|
||||
scope_max = 2;
|
||||
scope_min = 0;
|
||||
} else if (bits_input <= 8) {
|
||||
scope_max = 2;
|
||||
scope_min = -2;
|
||||
} else if (bits_output == 16) {
|
||||
scope_max = 5;
|
||||
scope_min = -5;
|
||||
} else {
|
||||
scope_max = 8;
|
||||
scope_min = -8;
|
||||
}
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
view, seed, scope_max, scope_min, 0);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Identity) {
|
||||
|
||||
cutlass::reference::host::TensorFillIdentity(view);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Gaussian) {
|
||||
|
||||
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Sequential) {
|
||||
|
||||
cutlass::reference::host::BlockFillSequential(
|
||||
view.data(), view.capacity());
|
||||
}
|
||||
else {
|
||||
// TODO: Implement the rest
|
||||
EXPECT_TRUE(false) << "Not implemented";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Initializes data structures
|
||||
void initialize(cutlass::gemm::GemmCoord problem_size) {
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
|
||||
tensor_A.resize(problem_size.mk());
|
||||
tensor_B.resize(problem_size.kn());
|
||||
tensor_C.resize(problem_size.mn());
|
||||
tensor_Z.resize(problem_size.mn());
|
||||
tensor_T.resize(problem_size.mn());
|
||||
tensor_Broadcast.resize({
|
||||
problem_size.m(),
|
||||
1
|
||||
});
|
||||
|
||||
tensor_C_ref.resize(problem_size.mn());
|
||||
tensor_Y_ref.resize(problem_size.mn());
|
||||
tensor_Z_ref.resize(problem_size.mn());
|
||||
tensor_T_ref.resize(problem_size.mn());
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_Broadcast.host_view(), init_C, seed + 2020));
|
||||
|
||||
// It is possible to randomly initialize to all zeros, so override this with non-zeros
|
||||
// in the upper left corner of each operand.
|
||||
tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
|
||||
tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
|
||||
tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
|
||||
|
||||
for (int m = 0; m < tensor_C_ref.extent().row(); ++m) {
|
||||
for (int n = 0; n < tensor_C_ref.extent().column(); ++n) {
|
||||
tensor_C_ref.at({m, n}) = ElementAccumulator(tensor_C.at({m, n}));
|
||||
}
|
||||
}
|
||||
|
||||
tensor_A.sync_device();
|
||||
tensor_B.sync_device();
|
||||
tensor_C.sync_device();
|
||||
tensor_Broadcast.sync_device();
|
||||
|
||||
tensor_Z.sync_device();
|
||||
tensor_T.sync_device();
|
||||
}
|
||||
|
||||
/// Compares computed reference with device reference and outputs to a file if incorrect
|
||||
bool compare_reference(
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
ElementAccumulator alpha,
|
||||
ElementAccumulator beta) {
|
||||
|
||||
tensor_Z.sync_host();
|
||||
tensor_T.sync_host();
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Z.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_T.host_view()), 0);
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Z_ref.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_T_ref.host_view()), 0);
|
||||
|
||||
bool passed = true;
|
||||
float norm_diff = 0;
|
||||
|
||||
if (OutputOp::kStoreZ) {
|
||||
norm_diff = cutlass::reference::host::TensorNormDiff(tensor_Z_ref.host_view(), tensor_Z.host_view(), float());
|
||||
passed = (norm_diff <= 0.1f);
|
||||
EXPECT_LT(norm_diff, 0.1f) << " tensor_Z is incorrect";
|
||||
}
|
||||
|
||||
if (OutputOp::kStoreT) {
|
||||
|
||||
norm_diff = cutlass::reference::host::TensorNormDiff(tensor_T_ref.host_view(), tensor_T.host_view(), float());
|
||||
passed = (passed && (norm_diff <= 0.1f));
|
||||
|
||||
EXPECT_LT(norm_diff, 0.1f) << " tensor_T is incorrect";
|
||||
}
|
||||
|
||||
|
||||
if (!passed) {
|
||||
|
||||
/*
|
||||
std::stringstream fname;
|
||||
|
||||
fname << "error_Gemm_device_"
|
||||
<< problem_size.m() << "x"
|
||||
<< problem_size.n() << "x"
|
||||
<< problem_size.k() << "_"
|
||||
<< Gemm::ThreadblockShape::kM << "x"
|
||||
<< Gemm::ThreadblockShape::kN << "x"
|
||||
<< Gemm::ThreadblockShape::kK << "_"
|
||||
<< Gemm::WarpShape::kM << "x"
|
||||
<< Gemm::WarpShape::kN << "x"
|
||||
<< Gemm::WarpShape::kK << ".txt";
|
||||
|
||||
std::ofstream file(fname.str());
|
||||
*/
|
||||
|
||||
std::ofstream file("errors_testbed_gemm_with_broadcast.txt");
|
||||
|
||||
|
||||
file
|
||||
<< "problem: " << problem_size
|
||||
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
|
||||
|
||||
file
|
||||
<< "A =\n" << tensor_A.host_view()
|
||||
<< "\nB =\n" << tensor_B.host_view()
|
||||
<< "\nC =\n" << tensor_C.host_view()
|
||||
<< "\nZ =\n" << tensor_Z.host_view()
|
||||
<< "\nT =\n" << tensor_T.host_view()
|
||||
<< "\n\n"
|
||||
<< "\nY_ref =\n" << tensor_Y_ref.host_view()
|
||||
<< "\nZ_ref =\n" << tensor_Z_ref.host_view()
|
||||
<< "\nT_ref =\n" << tensor_T_ref.host_view();
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/// Verifies the result is a GEMM
|
||||
bool verify(
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
ElementAccumulator alpha,
|
||||
ElementAccumulator beta) {
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
cutlass::reference::host::GemmComplex<
|
||||
typename Gemm::ElementA, typename Gemm::LayoutA,
|
||||
typename Gemm::ElementB, typename Gemm::LayoutB,
|
||||
ElementAccumulator, typename Gemm::LayoutC,
|
||||
ElementAccumulator, ElementAccumulator
|
||||
>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_A.host_ref(),
|
||||
Gemm::kTransformA,
|
||||
tensor_B.host_ref(),
|
||||
Gemm::kTransformB,
|
||||
beta,
|
||||
tensor_C_ref.host_ref(),
|
||||
tensor_Y_ref.host_ref(),
|
||||
ElementAccumulator(0)
|
||||
);
|
||||
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
|
||||
ReferenceOp reference_op;
|
||||
|
||||
|
||||
// compute tensor Z and tensor T
|
||||
for (int m = 0; m < problem_size.m(); ++m) {
|
||||
for (int n = 0; n < problem_size.n(); ++n) {
|
||||
|
||||
ElementZ z;
|
||||
ElementT t;
|
||||
|
||||
reference_op(z, t, tensor_Y_ref.at({m, n}), tensor_Broadcast.at({m, 0}));
|
||||
|
||||
tensor_Z_ref.at({m, n}) = z;
|
||||
tensor_T_ref.at({m, n}) = t;
|
||||
}
|
||||
}
|
||||
|
||||
return compare_reference(problem_size, alpha, beta);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerMultiprocessor < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
cutlass::gemm::GemmUniversalMode mode,
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
int batch_count = 1,
|
||||
ElementAccumulator alpha = ElementAccumulator(1),
|
||||
ElementAccumulator beta = ElementAccumulator(0)) {
|
||||
|
||||
// Waive test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
this->initialize(problem_size);
|
||||
|
||||
//
|
||||
// Initialize the GEMM operator
|
||||
//
|
||||
|
||||
typename Gemm::Arguments arguments{
|
||||
mode,
|
||||
problem_size,
|
||||
batch_count,
|
||||
{alpha, beta},
|
||||
tensor_A.device_data(),
|
||||
tensor_B.device_data(),
|
||||
tensor_C.device_data(),
|
||||
tensor_Z.device_data(),
|
||||
tensor_Broadcast.device_data(),
|
||||
tensor_T.device_data(),
|
||||
problem_size.m() * problem_size.k(),
|
||||
problem_size.n() * problem_size.k(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
tensor_A.layout().stride(0),
|
||||
tensor_B.layout().stride(0),
|
||||
tensor_C.layout().stride(0),
|
||||
tensor_Z.layout().stride(0),
|
||||
0, // This must be zero
|
||||
tensor_T.layout().stride(0),
|
||||
};
|
||||
|
||||
Gemm gemm_op;
|
||||
|
||||
size_t workspace_size = Gemm::get_workspace_size(arguments);
|
||||
|
||||
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
|
||||
|
||||
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
|
||||
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
|
||||
status = gemm_op();
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
bool passed = true;
|
||||
|
||||
passed = this->verify(problem_size, alpha, beta);
|
||||
|
||||
if (!passed) {
|
||||
std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
|
||||
}
|
||||
|
||||
//
|
||||
// Profile
|
||||
//
|
||||
|
||||
#if 0 // profiling disabled for now.
|
||||
|
||||
int const kWorkspaces = 100;
|
||||
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementA> profiling_tensor_A(tensor_A.capacity() * kWorkspaces);
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementB> profiling_tensor_B(tensor_B.capacity() * kWorkspaces);
|
||||
cutlass::DeviceAllocation<ElementC> profiling_tensor_C(tensor_C.capacity() * kWorkspaces);
|
||||
cutlass::DeviceAllocation<ElementC> profiling_tensor_Broadcast(tensor_Broadcast.capacity() * kWorkspaces);
|
||||
cutlass::DeviceAllocation<ElementZ> profiling_tensor_Z(tensor_Z.capacity() * kWorkspaces);
|
||||
cutlass::DeviceAllocation<ElementT> profiling_tensor_T(tensor_T.capacity() * kWorkspaces);
|
||||
|
||||
cudaEvent_t events[2];
|
||||
for (auto & event : events) {
|
||||
cudaError_t result = cudaEventCreate(&event);
|
||||
if (result != cudaSuccess) {
|
||||
EXPECT_EQ(result, cudaSuccess) << " cudaEventCreate() failed with error " << cudaGetErrorString(result);
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int const kWarmupIterations = 5;
|
||||
int const kProfilingIterations = 100;
|
||||
|
||||
for (int i = 0; i < kWarmupIterations; ++i) {
|
||||
status = gemm_op();
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
}
|
||||
|
||||
|
||||
cudaError_t result = cudaEventRecord(events[0]);
|
||||
EXPECT_EQ(result, cudaSuccess);
|
||||
|
||||
for (int i = 0; i < kProfilingIterations; ++i) {
|
||||
|
||||
typename Gemm::Arguments arguments{
|
||||
mode,
|
||||
problem_size,
|
||||
batch_count,
|
||||
{alpha, beta},
|
||||
profiling_tensor_A.get() + tensor_A.capacity() * (i % kWorkspaces),
|
||||
profiling_tensor_B.get() + tensor_B.capacity() * (i % kWorkspaces),
|
||||
profiling_tensor_C.get() + tensor_C.capacity() * (i % kWorkspaces),
|
||||
profiling_tensor_Z.get() + tensor_Z.capacity() * (i % kWorkspaces),
|
||||
profiling_tensor_Broadcast.get() + tensor_Broadcast.capacity() * (i % kWorkspaces),
|
||||
profiling_tensor_T.get() + tensor_T.capacity() * (i % kWorkspaces),
|
||||
problem_size.m() * problem_size.k(),
|
||||
problem_size.n() * problem_size.k(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
tensor_A.layout().stride(0),
|
||||
tensor_B.layout().stride(0),
|
||||
tensor_C.layout().stride(0),
|
||||
tensor_Z.layout().stride(0),
|
||||
0, // This must be zero
|
||||
tensor_T.layout().stride(0),
|
||||
};
|
||||
|
||||
gemm_op.initialize(arguments, workspace.get());
|
||||
status = gemm_op();
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[1]);
|
||||
EXPECT_EQ(result, cudaSuccess);
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
EXPECT_EQ(result, cudaSuccess);
|
||||
|
||||
float elapsed_time = 0;
|
||||
result = cudaEventElapsedTime(&elapsed_time, events[0], events[1]);
|
||||
EXPECT_EQ(result, cudaSuccess);
|
||||
|
||||
double average_time = double(elapsed_time) / double(kProfilingIterations);
|
||||
|
||||
std::cout << problem_size << ": " << average_time << " ms" << std::endl;
|
||||
|
||||
for (auto & event : events) {
|
||||
cudaEventDestroy(event);
|
||||
}
|
||||
#endif
|
||||
|
||||
return passed;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Gemm,
|
||||
typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
|
||||
>
|
||||
bool TestGemmWithBroadcast(
|
||||
cutlass::gemm::GemmCoord const & problem_size,
|
||||
cutlass::gemm::GemmUniversalMode mode,
|
||||
int batch_count,
|
||||
double alpha = 1.0,
|
||||
double beta = 2.0) {
|
||||
|
||||
bool passed = true;
|
||||
|
||||
TestbedGemmWithBroadcast<Gemm, ReferenceOp> testbed;
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
|
||||
passed = testbed.run(
|
||||
mode,
|
||||
problem_size,
|
||||
batch_count,
|
||||
cutlass::from_real<ElementAccumulator>(alpha),
|
||||
cutlass::from_real<ElementAccumulator>(beta)
|
||||
);
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Gemm,
|
||||
typename ReferenceOp = GemmWithBroadcastReferenceOp<Gemm>
|
||||
>
|
||||
bool TestAllGemmWithBroadcast() {
|
||||
|
||||
int M_problems[] = {8, 136, 264, 520};
|
||||
int N_problems[] = {8, 136, 264, 520};
|
||||
int K_problems[] = {8, 136, 264, 520};
|
||||
double alpha_problems[] = {1.25, 2.25};
|
||||
double beta_problems[] = {0, 1, 2.0};
|
||||
|
||||
bool passed = true;
|
||||
|
||||
for (int M : M_problems) {
|
||||
for (int N : N_problems) {
|
||||
for (int K : K_problems) {
|
||||
for (double alpha : alpha_problems) {
|
||||
for (double beta : beta_problems) {
|
||||
|
||||
TestbedGemmWithBroadcast<Gemm, ReferenceOp> testbed;
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
|
||||
passed = testbed.run(
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
{M, N, K},
|
||||
1,
|
||||
cutlass::from_real<ElementAccumulator>(alpha),
|
||||
cutlass::from_real<ElementAccumulator>(beta)
|
||||
);
|
||||
|
||||
EXPECT_TRUE(passed)
|
||||
<< "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta;
|
||||
|
||||
if (!passed) {
|
||||
|
||||
return passed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace device
|
||||
} // namespace gemm
|
||||
} // namespace test
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,491 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/distribution.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/tensor_norm.h"
|
||||
#include "cutlass/util/reference/host/gemm.h"
|
||||
#include "cutlass/util/reference/host/gemm_complex.h"
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemm, typename BinaryOp>
|
||||
struct GemmWithReductionReference {
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::ElementCompute;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
BinaryOp binary_op;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
GemmWithReductionReference() { }
|
||||
|
||||
ElementCompute operator()(
|
||||
ElementAccumulator d_y,
|
||||
ElementT t) {
|
||||
|
||||
return binary_op(ElementCompute(d_y), ElementCompute(t));
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Gemm,
|
||||
typename ReferenceOp
|
||||
>
|
||||
struct TestbedGemmWithReduction {
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
|
||||
|
||||
/// Initialization
|
||||
cutlass::Distribution::Kind init_A;
|
||||
cutlass::Distribution::Kind init_B;
|
||||
cutlass::Distribution::Kind init_C;
|
||||
uint64_t seed;
|
||||
|
||||
cutlass::HostTensor<typename Gemm::ElementA, typename Gemm::LayoutA> tensor_A;
|
||||
cutlass::HostTensor<typename Gemm::ElementB, typename Gemm::LayoutB> tensor_B;
|
||||
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_C;
|
||||
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> tensor_D;
|
||||
cutlass::HostTensor<typename Gemm::ElementAccumulator, typename Gemm::LayoutC> tensor_Reduction;
|
||||
cutlass::HostTensor<ElementT, typename Gemm::LayoutC> tensor_Tensor;
|
||||
cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> tensor_C_ref;
|
||||
cutlass::HostTensor<ElementAccumulator, typename Gemm::LayoutC> reference_d_Y;
|
||||
cutlass::HostTensor<typename Gemm::ElementC, typename Gemm::LayoutC> reference_D;
|
||||
cutlass::HostTensor<typename Gemm::ElementAccumulator, typename Gemm::LayoutC> reference_Reduction;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TestbedGemmWithReduction(
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = 2080
|
||||
):
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
/// Helper to initialize a tensor view
|
||||
template <typename Element, typename Layout>
|
||||
bool initialize_tensor(
|
||||
cutlass::TensorView<Element, Layout> view,
|
||||
cutlass::Distribution::Kind dist_kind,
|
||||
uint64_t seed) {
|
||||
|
||||
if (dist_kind == cutlass::Distribution::Uniform) {
|
||||
|
||||
double scope_max, scope_min;
|
||||
int bits_input = cutlass::sizeof_bits<Element>::value;
|
||||
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
|
||||
|
||||
if (bits_input == 1) {
|
||||
scope_max = 2;
|
||||
scope_min = 0;
|
||||
} else if (bits_input <= 8) {
|
||||
scope_max = 2;
|
||||
scope_min = -2;
|
||||
} else if (bits_output == 16) {
|
||||
scope_max = 5;
|
||||
scope_min = -5;
|
||||
} else {
|
||||
scope_max = 8;
|
||||
scope_min = -8;
|
||||
}
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
view, seed, scope_max, scope_min, 0);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Identity) {
|
||||
|
||||
cutlass::reference::host::TensorFillIdentity(view);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Gaussian) {
|
||||
|
||||
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
|
||||
}
|
||||
else if (dist_kind == cutlass::Distribution::Sequential) {
|
||||
|
||||
for (int m = 0; m < view.extent().row(); ++m) {
|
||||
for (int n = 0; n < view.extent().column(); ++n) {
|
||||
//view.at({m, n}) = Element(float(((idx ++) % 17) - 8));
|
||||
view.at({m, n}) = (n == 0 ? Element(m) : Element());
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// TODO: Implement the rest
|
||||
EXPECT_TRUE(false) << "Not implemented";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Initializes data structures
|
||||
void initialize(cutlass::gemm::GemmCoord problem_size) {
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
|
||||
tensor_A.resize(problem_size.mk());
|
||||
tensor_B.resize(problem_size.kn());
|
||||
tensor_C.resize(problem_size.mn());
|
||||
tensor_D.resize(problem_size.mn());
|
||||
|
||||
tensor_Reduction.resize({
|
||||
problem_size.m(),
|
||||
(problem_size.n() - 1 + Gemm::ThreadblockShape::kN) / Gemm::ThreadblockShape::kN
|
||||
});
|
||||
|
||||
tensor_Tensor.resize(problem_size.mn());
|
||||
reference_D.resize(problem_size.mn(), false);
|
||||
reference_d_Y.resize(problem_size.mn(), false);
|
||||
tensor_C_ref.resize(problem_size.mn(), false);
|
||||
reference_Reduction.resize({problem_size.m(), 1}, false);
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2019));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2018));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2017));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_Tensor.host_view(), init_C, seed + 2020));
|
||||
|
||||
// It is possible to randomly initialize to all zeros, so override this with non-zeros
|
||||
// in the upper left corner of each operand.
|
||||
tensor_A.host_view().at({0, 0}) = typename Gemm::ElementA(1);
|
||||
tensor_B.host_view().at({0, 0}) = typename Gemm::ElementB(1);
|
||||
tensor_C.host_view().at({0, 0}) = typename Gemm::ElementC(1);
|
||||
|
||||
for (int m = 0; m < tensor_C_ref.extent().row(); ++m) {
|
||||
for (int n = 0; n < tensor_C_ref.extent().column(); ++n) {
|
||||
tensor_C_ref.at({m, n}) = ElementAccumulator(tensor_C.at({m, n}));
|
||||
}
|
||||
}
|
||||
|
||||
tensor_A.sync_device();
|
||||
tensor_B.sync_device();
|
||||
tensor_C.sync_device();
|
||||
tensor_D.sync_device();
|
||||
tensor_Reduction.sync_device();
|
||||
tensor_Tensor.sync_device();
|
||||
}
|
||||
|
||||
/// Compares computed reference with device reference and outputs to a file if incorrect
|
||||
bool compare_reference(
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
ElementAccumulator alpha,
|
||||
ElementAccumulator beta) {
|
||||
|
||||
tensor_Reduction.sync_host();
|
||||
tensor_D.sync_host();
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Reduction.host_view()), 0);
|
||||
|
||||
bool passed = true;
|
||||
for (int m = 0; m < tensor_Reduction.extent().row(); ++m) {
|
||||
|
||||
ElementAccumulator reduced_value = ElementAccumulator();
|
||||
for (int j = 0; j < tensor_Reduction.extent().column(); ++j) {
|
||||
reduced_value += tensor_Reduction.at({m, j});
|
||||
}
|
||||
|
||||
if (reduced_value != reference_Reduction.at({m, 0})) {
|
||||
std::cout << "Error in bias[" << m << "] - Expected: " << reference_Reduction.at({m, 0}) << ", got: " << reduced_value << std::endl;
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
EXPECT_TRUE(passed) << "Reduction is incorect.";
|
||||
|
||||
if (!cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view())) {
|
||||
EXPECT_TRUE(false) << " mismatched reference";
|
||||
passed = false;
|
||||
}
|
||||
|
||||
if (!passed) {
|
||||
|
||||
/*
|
||||
std::stringstream fname;
|
||||
|
||||
fname << "error_Gemm_device_"
|
||||
<< problem_size.m() << "x"
|
||||
<< problem_size.n() << "x"
|
||||
<< problem_size.k() << "_"
|
||||
<< Gemm::ThreadblockShape::kM << "x"
|
||||
<< Gemm::ThreadblockShape::kN << "x"
|
||||
<< Gemm::ThreadblockShape::kK << "_"
|
||||
<< Gemm::WarpShape::kM << "x"
|
||||
<< Gemm::WarpShape::kN << "x"
|
||||
<< Gemm::WarpShape::kK << ".txt";
|
||||
|
||||
std::ofstream file(fname.str());
|
||||
*/
|
||||
|
||||
std::ofstream file("testbed_universal_errors_sm70.txt");
|
||||
|
||||
file
|
||||
<< "problem: " << problem_size
|
||||
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
|
||||
|
||||
file
|
||||
<< "A =\n" << tensor_A.host_view()
|
||||
<< "\nB =\n" << tensor_B.host_view()
|
||||
<< "\nC =\n" << tensor_C.host_view()
|
||||
<< "\nT = \n" << tensor_Tensor.host_view()
|
||||
<< "\n\nReference =\n" << reference_D.host_view()
|
||||
<< "\nComputed =\n" << tensor_D.host_view()
|
||||
<< "\n\nReduction =\n" << tensor_Reduction.host_view() << "\n"
|
||||
<< "\nReference reduction =\n" << reference_Reduction.host_view() << "\n";
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/// Verifies the result is a GEMM
|
||||
bool verify(
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
ElementAccumulator alpha,
|
||||
ElementAccumulator beta) {
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
cutlass::reference::host::GemmComplex<
|
||||
typename Gemm::ElementA, typename Gemm::LayoutA,
|
||||
typename Gemm::ElementB, typename Gemm::LayoutB,
|
||||
ElementAccumulator, typename Gemm::LayoutC,
|
||||
ElementAccumulator, ElementAccumulator
|
||||
>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_A.host_ref(),
|
||||
Gemm::kTransformA,
|
||||
tensor_B.host_ref(),
|
||||
Gemm::kTransformB,
|
||||
beta,
|
||||
tensor_C_ref.host_ref(),
|
||||
reference_d_Y.host_ref(),
|
||||
ElementAccumulator(0)
|
||||
);
|
||||
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
|
||||
ReferenceOp reference_op;
|
||||
|
||||
// compute backwards
|
||||
for (int m = 0; m < problem_size.m(); ++m) {
|
||||
ElementAccumulator reduced_value = ElementAccumulator();
|
||||
for (int n = 0; n < problem_size.n(); ++n) {
|
||||
ElementAccumulator d_full = reference_op(reference_d_Y.at({m, n}), tensor_Tensor.at({m, n}));
|
||||
reduced_value += d_full;
|
||||
reference_D.at({m, n}) = ElementC(d_full);
|
||||
}
|
||||
reference_Reduction.at({m, 0}) = reduced_value;
|
||||
}
|
||||
|
||||
return compare_reference(problem_size, alpha, beta);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerMultiprocessor < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
cutlass::gemm::GemmUniversalMode mode,
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
int batch_count = 1,
|
||||
ElementAccumulator alpha = ElementAccumulator(1),
|
||||
ElementAccumulator beta = ElementAccumulator(0)) {
|
||||
|
||||
// Waive test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
this->initialize(problem_size);
|
||||
|
||||
//
|
||||
// Initialize the GEMM operator
|
||||
//
|
||||
|
||||
typename Gemm::Arguments arguments{
|
||||
mode,
|
||||
problem_size,
|
||||
batch_count,
|
||||
{alpha, beta},
|
||||
tensor_A.device_data(),
|
||||
tensor_B.device_data(),
|
||||
tensor_C.device_data(),
|
||||
tensor_D.device_data(),
|
||||
tensor_Reduction.device_data(),
|
||||
tensor_Tensor.device_data(),
|
||||
problem_size.m() * problem_size.k(),
|
||||
problem_size.n() * problem_size.k(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
problem_size.m(),
|
||||
problem_size.m() * problem_size.n(),
|
||||
tensor_A.layout().stride(0),
|
||||
tensor_B.layout().stride(0),
|
||||
tensor_C.layout().stride(0),
|
||||
tensor_D.layout().stride(0),
|
||||
tensor_Reduction.layout().stride(0),
|
||||
tensor_Tensor.layout().stride(0),
|
||||
};
|
||||
|
||||
Gemm gemm_op;
|
||||
|
||||
size_t workspace_size = Gemm::get_workspace_size(arguments);
|
||||
|
||||
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
|
||||
|
||||
cutlass::Status status = gemm_op.initialize(arguments, workspace.get());
|
||||
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
|
||||
status = gemm_op();
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
|
||||
bool passed = this->verify(problem_size, alpha, beta);
|
||||
|
||||
if (!passed) {
|
||||
std::cout << "Failed with batch_count/split_k_slices = " << batch_count << std::endl;
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename Gemm, typename ReferenceOp>
|
||||
bool TestGemmWithReduction(
|
||||
cutlass::gemm::GemmCoord const & problem_size,
|
||||
cutlass::gemm::GemmUniversalMode mode,
|
||||
int batch_count = 1,
|
||||
double alpha = 1.0,
|
||||
double beta = 2.0) {
|
||||
|
||||
bool passed = true;
|
||||
|
||||
TestbedGemmWithReduction<Gemm, ReferenceOp> testbed;
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
|
||||
passed = testbed.run(
|
||||
mode,
|
||||
problem_size,
|
||||
batch_count,
|
||||
cutlass::from_real<ElementAccumulator>(alpha),
|
||||
cutlass::from_real<ElementAccumulator>(beta)
|
||||
);
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace device
|
||||
} // namespace gemm
|
||||
} // namespace test
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -103,8 +103,8 @@ public:
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
tensor_C.host_view(), seed * 2020, scope_max, scope_min, 0);
|
||||
|
||||
cutlass::reference::host::TensorFill(tensor_D.host_view());
|
||||
cutlass::reference::host::TensorFill(tensor_D_ref.host_view());
|
||||
cutlass::reference::host::TensorFill(tensor_D.host_view(), cutlass::complex<ElementC>());
|
||||
cutlass::reference::host::TensorFill(tensor_D_ref.host_view(), cutlass::complex<ElementC>());
|
||||
|
||||
tensor_A.sync_device();
|
||||
tensor_B.sync_device();
|
||||
@@ -162,10 +162,10 @@ public:
|
||||
ElementC *ptr_C = tensor_C.device_data();
|
||||
ElementC *ptr_D = tensor_D.device_data();
|
||||
|
||||
int lda = tensor_A.layout().stride(0);
|
||||
int ldb = tensor_B.layout().stride(0);
|
||||
int ldc = tensor_C.layout().stride(0);
|
||||
int ldd = tensor_D.layout().stride(0);
|
||||
typename LayoutA::Stride::Index lda = tensor_A.layout().stride(0);
|
||||
typename LayoutB::Stride::Index ldb = tensor_B.layout().stride(0);
|
||||
typename LayoutC::Stride::Index ldc = tensor_C.layout().stride(0);
|
||||
typename LayoutC::Stride::Index ldd = tensor_D.layout().stride(0);
|
||||
|
||||
int64_t imag_stride_A = tensor_A.imaginary_stride();
|
||||
int64_t imag_stride_B = tensor_B.imaginary_stride();
|
||||
@@ -266,15 +266,15 @@ template <typename Gemm>
|
||||
bool TestAllGemmPlanarComplex() {
|
||||
|
||||
int M[] = {
|
||||
16, 264,
|
||||
16, 64, 72, 144, 264, 520,
|
||||
};
|
||||
|
||||
int N[] = {
|
||||
16, 248,
|
||||
16, 64, 72, 144, 248, 264, 520
|
||||
};
|
||||
|
||||
int K[] = {
|
||||
8, 96,
|
||||
8, 64, 72, 96, 264, 520
|
||||
};
|
||||
|
||||
using ElementCompute = typename Gemm::EpilogueOutputOp::ElementCompute;
|
||||
|
||||
@@ -477,4 +477,3 @@ bool TestAllSparseGemm() {
|
||||
} // namespace test
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
Reference in New Issue
Block a user