leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

CUTLASS 3.5.1 (#1623)

Browse Source 
* CUTLASS 3.5.1

* updates, optimizations, fixes
This commit is contained in:
Vijay Thakkar
2024-07-29 08:46:24 -04:00
committed by GitHub
parent 56b46e2d13
commit be60a0b272
312 changed files with 19793 additions and 6775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/gemm/device/CMakeLists.txt
+13 -2
View File
@@ -310,6 +310,18 @@ cutlass_test_unit_add_executable(
sm90_gemm_tf32_tf32_f32_alignx_tensor_op_f32_warpspecialized_pingpong.cu
)
# Ptr Array test
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_sm90_ptr_array
sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array.cu
)
# Group Gemm test
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_sm90_group_gemm
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
)
# Fused epilogue tests
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_epilogue_fusion_sm90
@@ -348,7 +360,6 @@ cutlass_test_unit_add_executable(
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative.cu
sm90_gemm_f8_f8_f32_tensor_op_f32_cluster_warpspecialized_cooperative.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_tensorop_gmma_rs_warpspecialized_sm90
@@ -508,7 +519,7 @@ cutlass_test_unit_add_executable(
gemm_f8t_f8n_f32t_tensor_op_f32_sm89.cu
gemm_f8t_f8n_f32t_tensor_op_f32_sparse_sm89.cu
gemm_f8t_f8n_f8t_tensor_op_f32_sm89.cu
gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
# gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
)
cutlass_test_unit_add_executable(
test/unit/gemm/device/gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32_sm80.cu
+2 -2
View File
@@ -162,7 +162,7 @@ TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 64x64x16_32x32x16) {
TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -194,7 +194,7 @@ TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16)
TEST(SM80_Device_Gemm_cf32n_cf32t_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
test/unit/gemm/device/gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32_sm80.cu
+3 -3
View File
@@ -161,7 +161,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x64x16_32x32x16) {
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -193,7 +193,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x64x16_64x32x16)
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
@@ -225,7 +225,7 @@ TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 64x128x16_32x64x16)
TEST(SM80_Device_Gemm_cf32t_cf32n_cf32t_tensor_op_tf32_f32, 128x128x16_32x64x16) {
using Element = cutlass::complex<float>;;
using Element = cutlass::complex<float>;
using Gemm = cutlass::gemm::device::GemmComplex<
Element,
test/unit/gemm/device/gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
+1 -1
View File
@@ -79,7 +79,7 @@ TEST(SM80_Device_GemmUniversal_DirectStore_f16n_f16t_f32n_tensor_op_f32, 128x128
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
4, // This is the vector size of the epilogue.
4,
ElementAccumulator,
ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
test/unit/gemm/device/gemm_f16n_f16n_f16t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -91,7 +91,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -217,7 +217,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -235,7 +235,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -253,7 +253,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f16t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16n_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -73,7 +73,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -91,7 +91,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -164,7 +164,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -182,7 +182,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -200,7 +200,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -218,7 +218,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -236,7 +236,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -254,7 +254,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f16_sparse_sm80.cu
+14 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -217,7 +217,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -235,7 +235,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -253,7 +253,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f16t_tensor_op_f16, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f16n_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+212 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -198,7 +198,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -216,7 +216,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -234,7 +234,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -252,7 +252,7 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,205 @@ TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x32x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x32x128_64x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x128x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 32x256x128_32x64x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 128x16x128_32x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16n_f16t_f32t_tensor_op_f32, 256x16x128_64x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::ColumnMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED
test/unit/gemm/device/gemm_f16t_f16n_f16t_tensor_op_f16_sparse_sm80.cu
+13 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x256x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x128x64_64x64x64)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -109,7 +109,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x64x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -127,7 +127,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x256x64_64x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -145,7 +145,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x128x64_32x64x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -163,7 +163,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x64x64_64x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x64x64_32x32x64) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -199,7 +199,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x128x128_64x64x128
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -218,7 +218,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 256x64x128_64x64x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -236,7 +236,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 128x64x128_64x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -254,7 +254,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f16t_tensor_op_f16, 64x64x128_32x32x128)
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f16t_f16n_f32t_tensor_op_f32_sparse_sm80.cu
+213 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -198,7 +198,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x128x128_64x64x128
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -216,7 +216,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x64x128_64x64x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -234,7 +234,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x64x128_64x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -252,7 +252,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -266,6 +266,206 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 64x64x128_32x32x128)
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x128x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 32x256x128_32x64x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x32x128_32x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x32x128_64x32x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 128x16x128_32x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f16t_f16n_f32t_tensor_op_f32, 256x16x128_64x16x128) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 32>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED
test/unit/gemm/device/gemm_f16t_f16t_f32t_tensor_op_f32_sparse_sm80.cu
+9 -9
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -54,7 +54,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x256x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -72,7 +72,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 256x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -90,7 +90,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -108,7 +108,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -126,7 +126,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x256x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -144,7 +144,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x128x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -162,7 +162,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -180,7 +180,7 @@ TEST(SM80_Device_Sparse_Gemm_f16t_f16t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::half_t, cutlass::layout::RowMajor, cutlass::half_t,
cutlass::layout::RowMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
test/unit/gemm/device/gemm_f32n_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
test/unit/gemm/device/gemm_f32n_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+107 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
@@ -424,6 +424,100 @@ TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x128x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x256x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 32>,
cutlass::gemm::GemmShape<32, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f32n_f32t_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f32t_f32n_f32t_tensor_op_f32_sparse_sm80.cu
+357 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -423,6 +423,350 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 64x64x64_32x32x64) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x128x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 900)
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x256x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 32>,
cutlass::gemm::GemmShape<32, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
#endif
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 32x256x64_32x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 64>,
cutlass::gemm::GemmShape<32, 64, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x32x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 32>,
cutlass::gemm::GemmShape<32, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x32x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 64>,
cutlass::gemm::GemmShape<32, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x32x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 32>,
cutlass::gemm::GemmShape<64, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x32x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 64>,
cutlass::gemm::GemmShape<64, 32, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x16x32_32x16x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 32>,
cutlass::gemm::GemmShape<32, 16, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 128x16x64_32x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 64>,
cutlass::gemm::GemmShape<32, 16, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x16x32_64x16x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 32>,
cutlass::gemm::GemmShape<64, 16, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_f32t_f32n_f32t_tensor_op_f32, 256x16x64_64x16x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
cutlass::layout::ColumnMajor,
float,
cutlass::layout::RowMajor,
float,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 64>,
cutlass::gemm::GemmShape<64, 16, 64>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>,
6
>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_f32t_f32t_f32t_tensor_op_f32_sparse_sm80.cu
+13 -13
View File
@@ -35,7 +35,7 @@
#include <iostream>
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "../../common/cutlass_unit_test.h"
@@ -57,7 +57,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -88,7 +88,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x256x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -119,7 +119,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x128x32_64x64x32)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -150,7 +150,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x64x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -181,7 +181,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x256x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -212,7 +212,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -243,7 +243,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x128x32_32x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -274,7 +274,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x64x32_32x32x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -305,7 +305,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x128x64_64x64x64)
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -336,7 +336,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 256x64x64_64x64x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -367,7 +367,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 128x64x64_64x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
@@ -398,7 +398,7 @@ TEST(SM80_Device_Sparse_Gemm_f32t_f32t_f32t_tensor_op_f32, 64x64x64_32x32x64) {
using ElementOutput = float;
using ElementAccumulator = float;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
float,
cutlass::layout::RowMajor,
float,
test/unit/gemm/device/gemm_f8t_f8n_f32t_tensor_op_f32_sparse_sm89.cu
+5 -5
View File
@@ -41,7 +41,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -65,7 +65,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -89,7 +89,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe5m2n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -113,7 +113,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe4m3n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -137,7 +137,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_f32t_tensor_op_f32, 128x128x128_64x64
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
test/unit/gemm/device/gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
+47 -13
View File
@@ -43,7 +43,7 @@
#include "cutlass/cutlass.h"
#include "cutlass/epilogue/thread/activation.h"
#include "cutlass/epilogue/thread/linear_combination_generic_with_scaling.h"
#include "cutlass/gemm/device/gemm_sparse_with_absmax.h"
#include "cutlass/gemm/device/gemm_sparse_universal_with_absmax.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -78,7 +78,41 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
EpilogueOutputOp, cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, kStages
>;
bool passed = test::gemm::device::TestAllGemmWithAbsmax<Gemm, test::gemm::device::SparseTestbed<Gemm>, cutlass::epilogue::thread::Identity>();
EXPECT_TRUE(passed);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_f32t_tensor_op_f32, identity_128x128x128_64x64x128) {
// Test with float D and Aux for testing split-K without needing relative equality checks
using ElementA = cutlass::float_e4m3_t;
using ElementB = cutlass::float_e4m3_t;
using ElementOutput = float;
using ElementAuxOutput = ElementOutput;
using ElementAccumulator = float;
using LayoutA = cutlass::layout::RowMajor;
using LayoutB = cutlass::layout::ColumnMajor;
using LayoutC = cutlass::layout::RowMajor;
static int const kStages = 3;
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombinationGenericWithScalingAndAbsMax<
cutlass::epilogue::thread::Identity,
ElementOutput,
ElementAuxOutput,
128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator,
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -112,12 +146,12 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_fastac
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
EpilogueOutputOp, cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, kStages,
kAlignment, kAlignment, false, cutlass::arch::OpMultiplyAddFastAccum
kAlignment, kAlignment, cutlass::arch::OpMultiplyAddFastAccum
>;
bool passed = test::gemm::device::TestAllGemmWithAbsmax<Gemm, test::gemm::device::SparseTestbed<Gemm>, cutlass::epilogue::thread::Identity>();
@@ -146,7 +180,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, relu_128x128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -179,7 +213,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe5m2n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -212,7 +246,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -245,7 +279,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_fe4m3t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -278,7 +312,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe5m2t_tensor_op_f32, identity_128x12
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -311,7 +345,7 @@ TEST(SM89_Device_Sparse_Gemm_fe5m2t_fe5m2n_fe5m2t_tensor_op_f32, identity_diff_a
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -344,7 +378,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_128x64
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 64, 128>, cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -377,7 +411,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_noScal
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
@@ -414,7 +448,7 @@ TEST(SM89_Device_Sparse_Gemm_fe4m3t_fe4m3n_fe4m3t_tensor_op_f32, identity_noAux_
ElementAccumulator
>;
using Gemm = cutlass::gemm::device::SparseGemmWithAbsmax<
using Gemm = cutlass::gemm::device::GemmSparseUniversalWithAbsmax<
ElementA, LayoutA, ElementB, LayoutB, ElementOutput, LayoutC,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm89,
cutlass::gemm::GemmShape<128, 128, 128>, cutlass::gemm::GemmShape<64, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
test/unit/gemm/device/gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
+278 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -55,7 +55,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x256x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -74,7 +74,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -93,7 +93,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -113,7 +113,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x64x256_64x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -132,7 +132,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x256x256_64x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -151,7 +151,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x128x256_32x64x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -170,7 +170,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x256_64x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -189,7 +189,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -208,7 +208,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x128x512_64x64x512)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -228,7 +228,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x64x512_64x32x512) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -247,7 +247,7 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x512_32x32x512) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -261,7 +261,272 @@ TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 64x64x512_32x32x512) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x128x512_32x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 512>,
cutlass::gemm::GemmShape<32, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x256x256_32x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 256>,
cutlass::gemm::GemmShape<32, 64, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 32x256x512_32x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 512>,
cutlass::gemm::GemmShape<32, 64, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 16x128x512_16x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 512>,
cutlass::gemm::GemmShape<16, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 16x256x512_16x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 256, 512>,
cutlass::gemm::GemmShape<16, 64, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x32x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x32x512_32x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 512>,
cutlass::gemm::GemmShape<32, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x32x256_64x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 256>,
cutlass::gemm::GemmShape<64, 32, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x32x512_64x32x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 512>,
cutlass::gemm::GemmShape<64, 32, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x16x256_32x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 256>,
cutlass::gemm::GemmShape<32, 16, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 128x16x512_32x16x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 512>,
cutlass::gemm::GemmShape<32, 16, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x16x256_16x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 256>,
cutlass::gemm::GemmShape<64, 16, 256>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s4t_s4n_s32t_tensor_op_s32, 256x16x512_16x64x512) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
cutlass::int4b_t, cutlass::layout::RowMajor, cutlass::int4b_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 512>,
cutlass::gemm::GemmShape<64, 16, 512>, cutlass::gemm::GemmShape<16, 8, 128>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_s8t_s8n_s32t_tensor_op_s32_sparse_sm80.cu
+277 -13
View File
@@ -36,7 +36,7 @@
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/device/gemm_sparse.h"
#include "cutlass/gemm/device/gemm_sparse_universal.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
@@ -55,7 +55,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x256x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -74,7 +74,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x128x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -93,7 +93,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x128_64x64x128)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -113,7 +113,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x64x128_64x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -132,7 +132,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x256x128_64x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -151,7 +151,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x128x128_32x64x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -170,7 +170,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x128_64x32x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -189,7 +189,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x128_32x32x128) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -208,7 +208,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x128x256_64x64x256)
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -228,7 +228,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x64x256_64x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -247,7 +247,7 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::SparseGemm<
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
@@ -261,9 +261,273 @@ TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 64x64x256_32x32x256) {
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x128x128_32x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 128, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x256x128_32x64x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 128>,
cutlass::gemm::GemmShape<32, 64, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 32x256x256_32x64x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 256, 256>,
cutlass::gemm::GemmShape<32, 64, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 16x128x256_16x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 256>,
cutlass::gemm::GemmShape<16, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 16x128x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<16, 128, 256>,
cutlass::gemm::GemmShape<16, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x32x128_32x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 128>,
cutlass::gemm::GemmShape<32, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x32x256_32x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 32, 256>,
cutlass::gemm::GemmShape<32, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x32x128_64x32x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 128>,
cutlass::gemm::GemmShape<64, 32, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x32x256_64x32x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 32, 256>,
cutlass::gemm::GemmShape<64, 32, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x16x128_32x16x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 128>,
cutlass::gemm::GemmShape<32, 16, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 128x16x256_32x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 16, 256>,
cutlass::gemm::GemmShape<32, 16, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x16x128_64x16x128) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 128>,
cutlass::gemm::GemmShape<64, 16, 128>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
TEST(SM80_Device_Sparse_Gemm_s8t_s8n_s32t_tensor_op_s32, 256x16x256_64x16x256) {
using ElementOutput = int32_t;
using ElementAccumulator = int32_t;
using ElementCompute = int32_t;
using Gemm = cutlass::gemm::device::GemmSparseUniversal<
int8_t, cutlass::layout::RowMajor, int8_t,
cutlass::layout::ColumnMajor, ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<256, 16, 256>,
cutlass::gemm::GemmShape<64, 16, 256>, cutlass::gemm::GemmShape<16, 8, 64>,
cutlass::epilogue::thread::LinearCombinationClamp<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementCompute>,
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>, 6>;
EXPECT_TRUE(test::gemm::device::TestAllSparseGemm<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // defined(CUTLASS_ARCH_SPARSE_MMA_SM80_SUPPORTED)
test/unit/gemm/device/gemm_testbed_3x.hpp
+17 -10
View File
@@ -346,7 +346,7 @@ struct HostCollectiveMainloop {
tensor_A.resize(a_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(a_coord, stride_factor_A));
tensor_B.resize(b_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(b_coord, stride_factor_B));
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2022));
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2021));
@@ -710,7 +710,7 @@ struct HostCollectiveEpilogue {
using ActivationFunctor = non_void_t<typename FusionOp::ActivationFn,
cutlass::epilogue::thread::Identity<ElementCompute>>;
static constexpr bool IsBiasEnabled = FusionOp::IsPerRowBiasSupported;
static constexpr bool IsRowBiasEnabled = FusionOp::IsPerRowBiasSupported;
static constexpr bool IsDeBiasEnabled = FusionOp::IsDePerRowBiasSupported;
static constexpr bool IsPerRowScaleEnabled = FusionOp::IsPerRowScaleSupported;
static constexpr bool IsScaleFactorEnabled = FusionOp::IsScaleFactorSupported;
@@ -813,6 +813,7 @@ struct HostCollectiveEpilogue {
auto scalar_coord = cutlass::make_Coord(1);
auto col_vector_coord = cutlass::make_Coord(M);
auto row_vector_coord = cutlass::make_Coord(N);
if constexpr (IsPerRowScaleEnabled) {
alpha.resize(col_vector_coord);
EXPECT_TRUE(initialize_tensor(alpha.host_view(), init_scale, seed + 2023));
@@ -849,8 +850,10 @@ struct HostCollectiveEpilogue {
scale_D.sync_device();
}
if constexpr (IsBiasEnabled) {
bias.resize(col_vector_coord);
if constexpr (
IsRowBiasEnabled
) {
bias.resize(IsRowBiasEnabled ? col_vector_coord : row_vector_coord);
EXPECT_TRUE(initialize_tensor(bias.host_view(), init_bias, seed + 2023));
bias.sync_device();
}
@@ -1029,10 +1032,9 @@ struct HostCollectiveEpilogue {
file << "\n\n";
}
if constexpr (IsBiasEnabled) {
if constexpr (IsRowBiasEnabled) {
file << "\n\nBias = \n" << bias.host_view();
}
if constexpr (IsAuxInEnabled) {
file << "\n\nAux Input = \n" << tensor_Aux.host_view();
}
@@ -1090,7 +1092,9 @@ struct HostCollectiveEpilogue {
fusion_args.scale_d_ptr = scale_D.device_data();
}
if constexpr (IsBiasEnabled) {
if constexpr (
IsRowBiasEnabled
) {
fusion_args.bias_ptr = bias.device_data();
}
@@ -1153,7 +1157,7 @@ struct HostCollectiveEpilogue {
auto D = cute::make_tensor(detail::make_iterator(reference_D.host_data()),
cute::make_layout(cute::make_shape(M, N, L), stride_d));
auto Bias = cute::make_tensor(detail::make_iterator(IsDeBiasEnabled ? reference_dbias.host_data() : bias.host_data()),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cute::make_layout(cute::make_shape(IsRowBiasEnabled ? M : N)));
auto Aux = cute::make_tensor(detail::make_iterator(IsAuxInEnabled ? tensor_Aux.host_data() : reference_Aux.host_data()),
cute::make_layout(cute::make_shape(M, N, L), stride_Aux));
auto Valpha = cute::make_tensor(detail::make_iterator(alpha.host_data()),
@@ -1171,7 +1175,8 @@ struct HostCollectiveEpilogue {
decltype(Aux),
decltype(Valpha),
decltype(Vbeta),
ActivationFunctor
ActivationFunctor,
cutlass::plus<ElementCompute>
> epilogue_params{};
epilogue_params.C = C;
@@ -1186,7 +1191,9 @@ struct HostCollectiveEpilogue {
epilogue_params.scale_d = scale_D.at(coord_0);
}
if constexpr (IsBiasEnabled or IsDeBiasEnabled) {
if constexpr (IsRowBiasEnabled
or IsDeBiasEnabled)
{
epilogue_params.Bias = Bias;
}
test/unit/gemm/device/gemm_testbed_3x_ptr_array.hpp
+1792
View File
File diff suppressed because it is too large Load Diff
test/unit/gemm/device/sm90_evt_operations.hpp
+1 -8
View File
@@ -409,14 +409,7 @@ using Sm90LinCombPerColumnBias =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
Sm90RowBroadcast<
ceil_div(
EpilogueDescriptor::StagesC,
size(shape_div(take<0, 2>(typename EpilogueDescriptor::TileShape{}), typename EpilogueDescriptor::EpilogueTile{}))
) + 1,
typename EpilogueDescriptor::TileShape,
ElementBias
>
Sm90RowBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>
>
>;
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_aux_store.cu
+4 -8
View File
@@ -130,14 +130,10 @@ bool testEVTAuxStoreWithoutD() {
aux_store_D_block.reset(m * n);
Gemm gemm_op_base;
auto stride_A = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideA{}, cute::make_shape(m, k, cute::Int<1>{}));
auto stride_B = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideB{}, cute::make_shape(n, k, cute::Int<1>{}));
auto stride_C = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideC{}, cute::make_shape(m, n, cute::Int<1>{}));
auto stride_D = cutlass::make_cute_packed_stride(
typename GemmKernel::StrideD{}, cute::make_shape(m, n, cute::Int<1>{}));
auto stride_A = cutlass::make_cute_packed_stride(typename GemmKernel::StrideA{}, {m, k, 1});
auto stride_B = cutlass::make_cute_packed_stride(typename GemmKernel::StrideB{}, {n, k, 1});
auto stride_C = cutlass::make_cute_packed_stride(typename GemmKernel::StrideC{}, {m, n, 1});
auto stride_D = cutlass::make_cute_packed_stride(typename GemmKernel::StrideD{}, {m, n, 1});
auto arguments_base = typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
+120
View File
@@ -0,0 +1,120 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Ptr-Array GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/kernel/tile_scheduler.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp"
#include "cutlass/epilogue/collective/default_epilogue.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x_ptr_array.hpp"
#if defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
using namespace cute;
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_group_gemm, 128x128x64_2x2x1) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_256,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC *, AlignmentC,
ElementC, LayoutC *, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA *, AlignmentA,
ElementB, LayoutB *, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array.cu
+179
View File
@@ -0,0 +1,179 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide Ptr-Array GEMM interface
*/
#include <iostream>
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/gemm/kernel/gemm_universal.hpp"
#include "cutlass/gemm/kernel/tile_scheduler.hpp"
#include "cutlass/gemm/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp"
#include "cutlass/epilogue/collective/default_epilogue.hpp"
#include "cutlass/epilogue/thread/linear_combination.h"
#include "../../common/cutlass_unit_test.h"
#include "gemm_testbed_3x_ptr_array.hpp"
#if defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
using namespace cute;
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_ptr_array, 128x128x64_2x2x1) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC, AlignmentC,
ElementC, LayoutC, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA, AlignmentA,
ElementB, LayoutB, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool result = TestAll<Gemm>(1.0, 1.0);
EXPECT_TRUE(result);
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_ptr_array, 128x128x64_2x2x1_NoSmemEpi) {
// A matrix configuration
using ElementA = cutlass::half_t; // Element type for A matrix operand
using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand
constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes)
// B matrix configuration
using ElementB = cutlass::half_t; // Element type for B matrix operand
using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand
constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value; // Memory access granularity/alignment of B matrix in units of elements (up to 16 bytes)
// C/D matrix configuration
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands
constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes)
// Core kernel configurations
using ElementAccumulator = float; // Element type for internal accumulation
using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature
using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag
using TileShape = Shape<_128,_128,_64>; // Threadblock-level tile size
using ClusterShape = Shape<_2,_2,_1>; // Shape of the threadblocks in a cluster
using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based on the tile size
using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative; // Kernel to launch
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized; // Epilogue to launch
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
TileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAccumulator, ElementAccumulator,
ElementC, LayoutC, AlignmentC,
ElementC, LayoutC, AlignmentC,
EpilogueSchedule
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
ArchTag, OperatorClass,
ElementA, LayoutA, AlignmentA,
ElementB, LayoutB, AlignmentB,
ElementAccumulator,
TileShape, ClusterShape,
cutlass::gemm::collective::StageCountAutoCarveout<
static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
KernelSchedule
>::CollectiveOp;
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
CollectiveMainloop,
CollectiveEpilogue
>;
using namespace test::gemm::device;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(TestAll<Gemm>(1.0, 0.0));
}
#endif // defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_stream_k_scheduler.cu
+3 -2
View File
@@ -114,13 +114,14 @@ test_scheduler(
<< size<3>(problem_shape_mnkl)
<< " and grid size " << grid.x << "x"
<< grid.y << "x" << grid.z
<< " splits=" << params.splits_
<< " splits=" << params.divmod_splits_.divisor
<< " k_iter=" << params.divmod_tiles_per_output_tile_.divisor
<< " big_units_=" << params.big_units_
<< " big_groups_=" << params.big_groups_
<< " sk_tiles=" << params.sk_tiles_
<< " sk_units=" << params.sk_units_
<< " k_tiles_per_sk_unit=" << params.k_tiles_per_sk_unit_
<< " k_tiles_per_sk_unit=" << params.divmod_k_tiles_per_sk_unit_.divisor
<< " k_tiles_per_sk_big_unit=" << params.divmod_k_tiles_per_sk_big_unit_.divisor
<< " units_per_problem=" << params.units_per_problem_
<< " groups=" << params.divmod_sk_groups_.divisor << std::endl;
};
test/unit/gemm/device/testbed_sparse.h
+22 -21
View File
@@ -128,8 +128,8 @@ struct SparseTestbed {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
scope_max = 1;
scope_min = -1;
} else if (bits_output == 16) {
scope_max = 5;
scope_min = -5;
@@ -353,14 +353,25 @@ struct SparseTestbed {
//
typename Gemm::Arguments arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
tensor_A.device_ref(),
tensor_B.device_ref(),
tensor_C.device_ref(),
tensor_D.device_ref(),
tensor_E_reordered.device_ref(),
split_k_slices,
{alpha, beta},
split_k_slices
tensor_A.device_data(),
tensor_B.device_data(),
tensor_C.device_data(),
tensor_D.device_data(),
tensor_E_reordered.device_data(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
tensor_A.layout().stride(0),
tensor_B.layout().stride(0),
tensor_C.layout().stride(0),
tensor_D.layout().stride(0),
tensor_E_reordered.layout().stride(0)
};
Gemm gemm_op;
@@ -391,7 +402,7 @@ struct SparseTestbed {
bool passed = this->verify(problem_size, alpha, beta);
if (!passed) {
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << std::endl;
std::cout << "Error with split_k_slices = " << split_k_slices << ", alpha: " << alpha << ", beta: " << beta << ", m: " << problem_size.m() << ", n: " << problem_size.n() << ", k:" <<problem_size.k() << std::endl;
}
return passed;
@@ -420,11 +431,10 @@ bool TestAllSparseGemm() {
int problem_size_n[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int problem_size_k[] = {Gemm::ThreadblockShape::kK,
Gemm::ThreadblockShape::kK * (Gemm::kStages + 1)};
int problem_size_k[] = {Gemm::ThreadblockShape::kK * 8};
int split_k_slices[] = {
1, 2, 3
1, 2
};
double problem_alpha[] = {
@@ -444,17 +454,8 @@ bool TestAllSparseGemm() {
for (int k : problem_size_k) {
for (int split_k : split_k_slices) {
if (!Gemm::kSplitKSerial && split_k > 1) {
continue;
}
if (split_k > 1 && k / Gemm::ThreadblockShape::kK < split_k) {
continue;
}
for (auto alpha : problem_alpha) {
for (auto beta : problem_beta) {
cutlass::gemm::GemmCoord problem_size(m, n, k);
passed = testbed.run(
test/unit/gemm/device/testbed_with_absmax.h
+66 -27
View File
@@ -212,6 +212,9 @@ struct TestbedWithAmax {
EXPECT_GT(cutlass::reference::host::TensorNorm(underlying_testbed.tensor_D.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), underlying_testbed.tensor_D.host_view());
if (!passed) {
std::cout << "Comparison of D failed" << std::endl;
}
if (kScaleAux) {
tensor_Aux.sync_host();
@@ -219,14 +222,23 @@ struct TestbedWithAmax {
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_Aux.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(abs_max_Aux.host_view()), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_Aux.host_view()), 0);
passed &= cutlass::reference::host::TensorEquals(reference_Aux.host_view(), tensor_Aux.host_view());
passed &= cutlass::reference::host::TensorEquals(abs_max_Aux.host_view(), reference_abs_max_Aux.host_view());
if (!cutlass::reference::host::TensorEquals(reference_Aux.host_view(), tensor_Aux.host_view())) {
passed = false;
std::cout << "Comparison of Aux failed" << std::endl;
}
if (!cutlass::reference::host::TensorEquals(abs_max_Aux.host_view(), reference_abs_max_Aux.host_view())) {
passed = false;
std::cout << "Comparison of Aux absmax failed" << std::endl;
}
}
if (kScaleOutput) {
abs_max_D.sync_host();
EXPECT_GT(cutlass::reference::host::TensorNorm(abs_max_D.host_view()), 0);
passed &= cutlass::reference::host::TensorEquals(abs_max_D.host_view(), reference_abs_max_D.host_view());
if (!cutlass::reference::host::TensorEquals(abs_max_D.host_view(), reference_abs_max_D.host_view())) {
passed = false;
std::cout << "Comparison of D absmax failed" << std::endl;
}
}
EXPECT_TRUE(passed) << " mismatched reference";
@@ -417,16 +429,31 @@ struct TestbedWithAmax {
auto arguments = [&]() {
if constexpr (IsSparseTestbed) {
return typename Gemm::Arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
underlying_testbed.tensor_A.device_ref(),
underlying_testbed.tensor_B.device_ref(),
underlying_testbed.tensor_C.device_ref(),
underlying_testbed.tensor_D.device_ref(),
underlying_testbed.tensor_E_reordered.device_ref(),
tensor_Aux.device_ref(),
batch_count,
epilogue_params,
underlying_testbed.tensor_A.device_data(),
underlying_testbed.tensor_B.device_data(),
underlying_testbed.tensor_C.device_data(),
underlying_testbed.tensor_D.device_data(),
underlying_testbed.tensor_E_reordered.device_data(),
tensor_Aux.device_data(),
tensor_Vector.device_data(),
0, // stride vector
epilogue_params
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
int64_t(),
underlying_testbed.tensor_A.layout().stride(0),
underlying_testbed.tensor_B.layout().stride(0),
underlying_testbed.tensor_C.layout().stride(0),
underlying_testbed.tensor_D.layout().stride(0),
underlying_testbed.tensor_E_reordered.layout().stride(0),
tensor_Aux.layout().stride(0),
0 // stride vector
};
}
else {
@@ -522,35 +549,47 @@ bool TestAllGemmWithAbsmax(bool scaleA=true, bool scaleB=true, bool scaleC=true)
int M_problems[] = {kAlignmentM, 128 + 32};
int N_problems[] = {kAlignmentN, 512 - 2 * kAlignmentN};
int K_problems[] = {Gemm::ThreadblockShape::kK, Gemm::ThreadblockShape::kK * (Gemm::kStages + 1)};
int K_problems[] = {Gemm::ThreadblockShape::kK * 2};
double alpha_problems[] = {1.};
double beta_problems[] = {0.};
int split_k_slices[] = {
1, 2
};
bool passed = true;
for (int M : M_problems) {
for (int N : N_problems) {
for (int K : K_problems) {
for (double alpha : alpha_problems) {
for (double beta : beta_problems) {
TestbedWithAmax<Gemm, GemmTestbed, ActivationFunctor> testbed(scaleA, scaleB, scaleC);
for (int split_k : split_k_slices) {
if (cutlass::sizeof_bits_v<typename Gemm::EpilogueOutputOp::ElementOutput> <= 8 && split_k > 1) {
// Don't test split-K with FP8 output. The kernel being tested will writie partial accumulations
// for different splits to global memory in FP8, while the reference kernel will not. This leads
// to mismatches that are difficult to capture without a permissive relative equality check threshold.
continue;
}
using ElementAccumulator = typename Gemm::ElementAccumulator;
for (double alpha : alpha_problems) {
for (double beta : beta_problems) {
TestbedWithAmax<Gemm, GemmTestbed, ActivationFunctor> testbed(scaleA, scaleB, scaleC);
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
{M, N, K},
1,
cutlass::from_real<ElementAccumulator>(alpha),
cutlass::from_real<ElementAccumulator>(beta)
);
using ElementAccumulator = typename Gemm::ElementAccumulator;
EXPECT_TRUE(passed)
<< "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta;
passed = testbed.run(
cutlass::gemm::GemmUniversalMode::kGemm,
{M, N, K},
split_k,
cutlass::from_real<ElementAccumulator>(alpha),
cutlass::from_real<ElementAccumulator>(beta)
);
if (!passed) {
EXPECT_TRUE(passed)
<< "M: " << M << ", N: " << N << ", K: " << K << ", alpha: " << alpha << ", beta: " << beta << ", split_k:" << split_k;
return passed;
if (!passed) {
return passed;
}
}
}
}
test/unit/gemm/threadblock/mma_multistage_sparse.cu
+105
View File
@@ -179,6 +179,111 @@ TEST(SM80_sparse_gemm_threadblock_congruous,
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x32x64_32x32x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(128, 32, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<128, 32, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_32x256x128_32x64x128_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(32, 256, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<32, 256, 128>;
using WarpShape = cutlass::gemm::GemmShape<32, 64, 128>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x16x64_32x16x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
using LayoutA = cutlass::layout::ColumnMajor;
using ElementB = cutlass::half_t;
using LayoutB = cutlass::layout::RowMajor;
using ElementC = float;
using LayoutC = cutlass::layout::ColumnMajor;
cutlass::gemm::GemmCoord problem_size(128, 16, 512);
using ThreadblockShape = cutlass::gemm::GemmShape<128, 16, 64>;
using WarpShape = cutlass::gemm::GemmShape<32, 16, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
float alpha = 1.f;
float beta = 0.0f;
int const Stages = 4;
// Define the MmaCore components
using MmaCore = typename cutlass::gemm::threadblock::DefaultSparseMmaCore<
ThreadblockShape, WarpShape, InstructionShape, ElementA, LayoutA,
ElementB, LayoutB, ElementC, LayoutC, cutlass::arch::OpClassTensorOp,
Stages>;
dim3 grid(1, 1);
dim3 block(32, 4, 1);
test::gemm::threadblock::SparseTestbed<MmaCore>(
problem_size.m(), problem_size.n(), problem_size.k(), alpha, beta)
.run(grid, block);
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_sparse_gemm_threadblock_congruous,
tensor_op_128x128x64_64x64x64_16x8x32_4stage) {
using ElementA = cutlass::half_t;
test/unit/gemm/warp/gemm_mixed_input_sm80.cu
+1 -1
View File
@@ -104,7 +104,7 @@ TEST(SM80_warp_gemm_mixed_input_tensor_op_crosswise_i8_f16, 128x128x64_64x64x64_
using Shape = cutlass::gemm::GemmShape<64, 64, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 16>;
using ElementA = int8_t;
using ElementB = cutlass::half_t;;
using ElementB = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
cutlass::sizeof_bits<ElementA>::value, 64>;
test/unit/gemm/warp/gemm_sparse_sm80.cu
+60 -18
View File
@@ -327,6 +327,48 @@ TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 128x128x64_32x32x64_16x8x32)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 64x32x64_32x32x64_16x8x32) {
using Shape = cutlass::gemm::GemmShape<32, 32, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
using Element = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 32>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 64> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 64x16x64_32x16x64_16x8x32) {
using Shape = cutlass::gemm::GemmShape<32, 16, 64>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
using Element = cutlass::half_t;
using ElementC = float;
using LayoutA = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 64>;
using LayoutB = cutlass::layout::RowMajorTensorOpMultiplicandCongruous<
cutlass::sizeof_bits<Element>::value, 16>;
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 16, 64> >()
.run();
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_sparse_tensor_op_congruous_f16, 128x64x128_64x32x128_16x8x32) {
using Shape = cutlass::gemm::GemmShape<64, 32, 128>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>;
@@ -402,7 +444,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_64x64x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -423,7 +465,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_64x32x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -444,7 +486,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x64x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -465,7 +507,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x32x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -486,7 +528,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x128x128_32x16x128_16x8x64
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 128> >()
@@ -507,7 +549,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 128x64x256_64x32x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 64, 256> >()
@@ -528,7 +570,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x128x256_32x64x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 128, 256> >()
@@ -549,7 +591,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x64x256_32x32x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 64, 256> >()
@@ -570,7 +612,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s8, 64x32x256_32x16x256_16x8x64)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 256> >()
@@ -927,7 +969,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_64x64x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -948,7 +990,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_64x32x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -969,7 +1011,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x64x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -990,7 +1032,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x32x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -1011,7 +1053,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x128x256_32x16x256_16x8x12
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 128, 256> >()
@@ -1032,7 +1074,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 128x64x512_64x32x512_16x8x128
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<128, 64, 512> >()
@@ -1053,7 +1095,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x128x512_32x64x512_16x8x128
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 128, 512> >()
@@ -1074,7 +1116,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x64x512_32x32x512_16x8x128)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 64, 512> >()
@@ -1095,7 +1137,7 @@ TEST(SM80_warp_gemm_sparse_tensor_op_crosswise_s4, 64x32x512_32x16x512_16x8x128)
using MmaTensorOp = typename cutlass::gemm::warp::DefaultSparseMmaTensorOp<
Shape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementC,
cutlass::layout::RowMajor>::Type;
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
test::gemm::warp::SparseTestbed<MmaTensorOp,
cutlass::gemm::GemmShape<64, 32, 512> >()