v4.2 release. (#2587)
* Fix default cluster callback values to 1 to avoid profiler failure when these values are not set in command line. * v4.2 release.
This commit is contained in:
Junkai-Wu
@@ -33,7 +33,8 @@
|
||||
*/
|
||||
|
||||
#include <complex>
|
||||
#include <cuda/std/complex>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include CUDA_STD_HEADER(complex)
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
|
||||
|
||||
@@ -609,6 +609,8 @@ TEST(SM89_CuTe_Ada, CooperativeGemm_e5m2e5m2f32_MMA) {
|
||||
test_cooperative_gemm_col_major_layout<thread_block_size, MaxVecBits, TA, TB, TC>(shape_mnk, tiled_mma);
|
||||
}
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ > 12) || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 8)
|
||||
|
||||
TEST(SM89_CuTe_Ada, CooperativeGemm_e4m3e4m3f16_MMA) {
|
||||
using TA = cutlass::float_e4m3_t;
|
||||
using TB = cutlass::float_e4m3_t;
|
||||
@@ -680,3 +682,5 @@ TEST(SM89_CuTe_Ada, CooperativeGemm_e5m2e5m2f16_MMA) {
|
||||
|
||||
test_cooperative_gemm_col_major_layout<thread_block_size, MaxVecBits, TA, TB, TC>(shape_mnk, tiled_mma);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -44,7 +44,7 @@ __global__ void
|
||||
movm_test_device(uint16_t* g_in, uint16_t* g_out)
|
||||
{
|
||||
int tid = threadIdx.x;
|
||||
|
||||
|
||||
// load input gmem -> register
|
||||
uint32_t reg = reinterpret_cast<uint32_t*>(g_in)[tid];
|
||||
|
||||
@@ -128,7 +128,7 @@ TEST(SM75_CuTe_Turing, Movm)
|
||||
//
|
||||
// CuTe MOVM
|
||||
//
|
||||
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
|
||||
@@ -345,6 +345,13 @@ cutlass_test_unit_gemm_device_add_executable(
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
|
||||
)
|
||||
|
||||
# Blockwise Gemm test
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90_blockwise
|
||||
|
||||
sm90_gemm_f8_f8_f8_tensor_op_f32_blockwise.cu
|
||||
)
|
||||
|
||||
# Sparse tests
|
||||
# Sparse kernels trigger an ICE in gcc 7.5
|
||||
if (NOT (CUTLASS_GNU_HOST_COMPILE AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0))
|
||||
@@ -801,7 +808,7 @@ cutlass_test_unit_gemm_device_add_executable(
|
||||
hemm_cf64_cf64_cf64_tensor_op_f64_sm90.cu
|
||||
)
|
||||
|
||||
if (NOT CUTLASS_NVCC_ARCHS MATCHES 101|101a|101f|103|103a|103f)
|
||||
if (NOT CUTLASS_NVCC_ARCHS MATCHES 100f|101|101a|101f|103|103a|103f)
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_blas3_gaussian
|
||||
|
||||
@@ -947,6 +954,219 @@ cutlass_test_unit_gemm_device_add_executable(
|
||||
sm100_gemm_f8_f8_f8_tensor_op_f32_blockwise.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_runtime_datatype_alignx_sm100
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm100_gemm_f8_f8_f8_tensor_op_f32_runtime_datatype_alignx.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_alignx_sm100
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
sm100_gemm_f8_f8_f8_tensor_op_f32_alignx.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_alignx_streamK_sm100
|
||||
|
||||
# setting batch size fo 1 to control memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm100_gemm_f8_f8_f8_tensor_op_f32_alignx_streamK.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
if (CUTLASS_NVCC_ARCHS MATCHES 103a|103f)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_1sm
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_1sm.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_f4_tensorop_sm103_nosmem
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_tensor_op_f32_nosmem.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_2sm
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_2sm.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_group_1sm_128x128
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_group_1sm_128x128.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_group_1sm_128x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_group_1sm_128x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_group_2sm_256x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_group_2sm_256x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_group_2sm_256x256
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_group_2sm_256x256.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_ptr_array_1sm_128x128
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_ptr_array_1sm_128x128.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_ptr_array_1sm_128x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_ptr_array_1sm_128x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_ptr_array_2sm_256x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_ptr_array_2sm_256x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_ptr_array_2sm_256x256
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_ptr_array_2sm_256x256.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_streamk
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_stream_k.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_2sm_256x256
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_2sm_256x256.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_2sm_256x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_2sm_256x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_2sm_256x128
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_2sm_256x128.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_1sm_128x128
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_1sm_128x128.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_1sm_128x192
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_1sm_128x192.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm103_1sm_128x256
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm103_gemm_f4_f4_f32_tensor_op_f32_1sm_128x256.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
|
||||
|
||||
@@ -1219,6 +1219,33 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedCooperative
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_, stride_factor_B_) {}
|
||||
};
|
||||
|
||||
//
|
||||
// Block Scaled Gemm Input Operands : A , B, scalefactorA, scalefactorB
|
||||
//
|
||||
template<
|
||||
class Gemm,
|
||||
int SchedulerPipelineStageCount_,
|
||||
int AccumulatorPipelineStageCount_,
|
||||
class ElementA_,
|
||||
class ElementB_
|
||||
>
|
||||
struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaledSm103<SchedulerPipelineStageCount_,
|
||||
AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_> : public
|
||||
HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaledSm100<SchedulerPipelineStageCount_,AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_> {
|
||||
using Base = HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaledSm100<SchedulerPipelineStageCount_,AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_>;
|
||||
HostCollectiveMainloop(
|
||||
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = Base::kDefaultSeed,
|
||||
typename Base::LayoutTagA::Stride stride_factor_A_ = typename Base::LayoutTagA::Stride(),
|
||||
typename Base::LayoutTagB::Stride stride_factor_B_ = typename Base::LayoutTagB::Stride()
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_, stride_factor_B_) {}
|
||||
};
|
||||
|
||||
//
|
||||
// Block Scaled Structured Sparse Gemm Input Operands : A_compressed, B, metadata, scalefactorA, scalefactorB
|
||||
//
|
||||
|
||||
@@ -830,6 +830,32 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCoo
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_, stride_factor_B_) {}
|
||||
};
|
||||
|
||||
//
|
||||
// Block Scaled Gemm Input Operands : A , B, scalefactorA, scalefactorB
|
||||
//
|
||||
template<
|
||||
class Gemm,
|
||||
int SchedulerPipelineStageCount_,
|
||||
int AccumulatorPipelineStageCount_,
|
||||
class ElementA_,
|
||||
class ElementB_
|
||||
>
|
||||
struct HostCollectiveMainloop<cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockScaledSm103<SchedulerPipelineStageCount_,
|
||||
AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_> : public
|
||||
HostCollectiveMainloop<cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockScaledSm100<SchedulerPipelineStageCount_,AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_> {
|
||||
using Base = HostCollectiveMainloop<cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockScaledSm100<SchedulerPipelineStageCount_,AccumulatorPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_>;
|
||||
HostCollectiveMainloop(
|
||||
CheckEquality check_relative_equality_ = CheckEquality::EXACT,
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = Base::kDefaultSeed,
|
||||
typename Base::LayoutTagA::Stride stride_factor_A_ = typename Base::LayoutTagA::Stride(),
|
||||
typename Base::LayoutTagB::Stride stride_factor_B_ = typename Base::LayoutTagB::Stride()
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_, stride_factor_B_) {}
|
||||
};
|
||||
|
||||
template<class Gemm>
|
||||
struct HostCollectiveDefaultEpilogue {
|
||||
|
||||
@@ -59,6 +59,56 @@ using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
TEST(SM100_Device_Gemm_e2m1t_e2m1n_f32n_tensorop_1sm_f32_group_nosmem, 512x256x256_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementA = cutlass::float_e2m1_t;
|
||||
using ElementB = cutlass::float_e2m1_t;
|
||||
using ElementC = float;
|
||||
using ElementD = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
using ElementSF = cutlass::float_ue8m0_t;
|
||||
using MmaTypePairA = cute::tuple<ElementA, ElementSF>;
|
||||
using MmaTypePairB = cute::tuple<ElementB, ElementSF>;
|
||||
|
||||
using MmaTileShape = cute::Shape<_128,_128,_256>;
|
||||
using ClusterShape = Shape<_4,_2,_1>;
|
||||
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized1Sm;
|
||||
using MainloopSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmMxf4Sm100;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAccumulator, ElementCompute,
|
||||
ElementC, LayoutC *, 16 / sizeof(ElementC),
|
||||
ElementD, LayoutC *, 16 / sizeof(ElementC),
|
||||
EpilogueSchedule
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
MmaTypePairA, LayoutA *, 32,
|
||||
MmaTypePairB, LayoutB *, 32,
|
||||
ElementAccumulator,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
MainloopSchedule
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e2m1t_e2m1n_f32n_tensorop_1sm_f32_group, 512x256x256_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
@@ -209,6 +259,56 @@ TEST(SM100Only_Device_Gemm_e2m1t_e2m1n_f32n_tensorop_1sm_f32_group, 256x512x256_
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100Only_Device_Gemm_e2m1t_e2m1n_f32n_tensorop_2sm_f32_group_nosmem, 256x256x256_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementA = cutlass::float_e2m1_t;
|
||||
using ElementB = cutlass::float_e2m1_t;
|
||||
using ElementC = float;
|
||||
using ElementD = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
using ElementSF = cutlass::float_ue8m0_t;
|
||||
using MmaTypePairA = cute::tuple<ElementA, ElementSF>;
|
||||
using MmaTypePairB = cute::tuple<ElementB, ElementSF>;
|
||||
|
||||
using MmaTileShape = cute::Shape<_256,_128,_256>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using EpilogueSchedule = cutlass::epilogue::PtrArrayNoSmemWarpSpecialized2Sm;
|
||||
using MainloopSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmMxf4Sm100;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAccumulator, ElementCompute,
|
||||
ElementC, LayoutC *, 4,
|
||||
ElementD, LayoutC *, 4,
|
||||
EpilogueSchedule
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
MmaTypePairA, LayoutA *, 32,
|
||||
MmaTypePairB, LayoutB *, 32,
|
||||
ElementAccumulator,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
MainloopSchedule
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.5);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100Only_Device_Gemm_e2m1t_e2m1n_f32n_tensorop_2sm_f32_group, 256x256x256_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3n_e4m3t_e4m3t_tensorop_1sm_f32_align4, 64x128x64_1x1x1) {
|
||||
using MmaTileShape = Shape<_64,_128,_64>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cutlass::layout::ColumnMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestSmall<Gemm, true>(1.0, 0.5);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3n_e4m3t_e4m3t_tensorop_1sm_f32_align4, 128x128x64_1x1x1) {
|
||||
using MmaTileShape = Shape<_128,_128,_64>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cutlass::layout::ColumnMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestSmall<Gemm, true>(1.0, 0.5);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_1sm_f32_align4, 64x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_64,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::ColumnMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestSmall<Gemm, true>(1.0, 0.5);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_1sm_f32_align4, 128x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_128,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::ColumnMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestSmall<Gemm, true>(1.0, 0.5);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
@@ -0,0 +1,98 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3n_e4m3t_e4m3t_tensorop_1sm_f32_align4_StreamK, 64x128x64_1x1x1) {
|
||||
using MmaTileShape = Shape<_64,_128,_64>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cutlass::layout::ColumnMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
cutlass::gemm::StreamKScheduler>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestSmall<Gemm, true>(1.0, 0.5, CheckEquality::RELATIVE, ScalarLoc::ON_DEVICE, VectorScale::ENABLED, {256 + 4});
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
@@ -303,6 +303,7 @@ TEST(SM100_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_2sm_f32_align16_blockwise, 256
|
||||
|
||||
}
|
||||
|
||||
|
||||
TEST(SM100_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_2sm_f32_align16_blockwise, 256x128x128_2x1x1_64x64x64_scale) {
|
||||
|
||||
bool passed = groupwise_test<UMMA::Major::MN, UMMA::Major::MN>(
|
||||
@@ -317,4 +318,5 @@ TEST(SM100_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_2sm_f32_align16_blockwise, 256
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
TEST(SM100_Device_Gemm_e5m2t_e4m3n_e4m3t_tensorop_1sm_f32_runtime_datatype_align8, 64x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_64,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::ColumnMajor, 8,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestRuntimeDataTypeSmall<Gemm, cute::UMMA::MXF8F6F4Format, cute::UMMA::MXF8F6F4Format, true>(cute::UMMA::MXF8F6F4Format::E5M2, cute::UMMA::MXF8F6F4Format::E4M3);
|
||||
EXPECT_TRUE(pass);
|
||||
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e5m2t_e4m3n_e4m3t_tensorop_1sm_f32_runtime_datatype_align8, 128x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_128,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::RowMajor, 8,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::ColumnMajor, 8,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestRuntimeDataTypeSmall<Gemm, cute::UMMA::MXF8F6F4Format, cute::UMMA::MXF8F6F4Format, true>(cute::UMMA::MXF8F6F4Format::E5M2, cute::UMMA::MXF8F6F4Format::E4M3);
|
||||
EXPECT_TRUE(pass);
|
||||
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e5m2t_e4m3n_e4m3t_tensorop_1sm_f32_runtime_datatype_align4, 64x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_64,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::ColumnMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestRuntimeDataTypeSmall<Gemm, cute::UMMA::MXF8F6F4Format, cute::UMMA::MXF8F6F4Format, true>(cute::UMMA::MXF8F6F4Format::E5M2, cute::UMMA::MXF8F6F4Format::E4M3);
|
||||
EXPECT_TRUE(pass);
|
||||
|
||||
}
|
||||
|
||||
TEST(SM100_Device_Gemm_e5m2t_e4m3n_e4m3t_tensorop_1sm_f32_runtime_datatype_align4, 128x128x128_1x1x1) {
|
||||
using MmaTileShape = Shape<_128,_128,_128>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::float_e4m3_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::type_erased_dynamic_float8_t, cutlass::layout::ColumnMajor, 4,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
void>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = TestRuntimeDataTypeSmall<Gemm, cute::UMMA::MXF8F6F4Format, cute::UMMA::MXF8F6F4Format, true>(cute::UMMA::MXF8F6F4Format::E5M2, cute::UMMA::MXF8F6F4Format::E4M3);
|
||||
EXPECT_TRUE(pass);
|
||||
|
||||
}
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
@@ -44,6 +44,7 @@ cutlass_test_unit_gemm_device_add_executable(
|
||||
|
||||
f16_f16_void_f32.cu
|
||||
f16_f16_f16_f16_fusion.cu
|
||||
f16_f16_void_f32_narrow_mma_n.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
@@ -54,6 +55,7 @@ cutlass_test_unit_gemm_device_add_executable(
|
||||
|
||||
f8_f8_void_f32.cu
|
||||
f8_f8_f16_f8_fusion.cu
|
||||
f8_f8_void_bf16_narrow_mma_n.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "../gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// FP16T x FP16N -> FP32 with 64x8x16 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_f16t_f16n_void_f32n_tensor_op_f32, 64x8x64_2x2x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_64,_8,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16N x FP16T -> FP32 with 64x8x16 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_f16n_f16t_void_f32n_tensor_op_f32, 64x8x64_2x2x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_64,_8,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16N x FP16T -> FP32 with 128x8x16 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_f16n_f16t_void_f32n_tensor_op_f32, 128x8x64_2x2x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_8,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16T x FP16N -> FP32 with 128x16x16 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_f16t_f16n_void_f32n_tensor_op_f32, 128x16x64_2x2x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_16,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16N x FP16T -> FP32 with 128x16x16 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_f16n_f16t_void_f32n_tensor_op_f32, 128x16x64_2x2x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_16,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16T x FP16N -> FP32 with 256x16x16 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_f16t_f16n_void_f32n_tensor_op_f32, 256x16x64_2x2x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_256,_16,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP16N x FP16T -> FP32 with 256x16x16 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_f16n_f16t_void_f32n_tensor_op_f32, 256x16x64_2x2x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_256,_16,_64>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_2,_2,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::half_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::half_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = float;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,922 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "../gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// FP8T x FP8N -> FP32 with 64x8x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3t_e4m3n_void_bf16n_tensor_op_f32, 64x8x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_64,_8,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8N -> FP32 with 64x8x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3n_void_bf16n_tensor_op_f32, 64x8x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_64,_8,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8T -> FP32 with 64x16x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3t_void_bf16n_tensor_op_f32, 64x16x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_64,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8T x FP8N -> FP32 with 128x8x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3t_e4m3n_void_bf16n_tensor_op_f32, 128x8x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_8,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8N -> FP32 with 128x8x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3n_void_bf16n_tensor_op_f32, 128x8x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_8,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8T -> FP32 with 128x16x32 MMA-1CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3t_void_bf16n_tensor_op_f32, 128x16x128_4x1x1_1sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8T x FP8N -> FP32 with 128x16x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3t_e4m3n_void_bf16n_tensor_op_f32, 128x16x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8N -> FP32 with 128x16x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3n_void_bf16n_tensor_op_f32, 128x16x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8T -> FP32 with 128x32x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3t_void_bf16n_tensor_op_f32, 128x32x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_128,_32,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8T x FP8N -> FP32 with 256x16x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3t_e4m3n_void_bf16n_tensor_op_f32, 256x16x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_256,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::RowMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8N -> FP32 with 256x16x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3n_void_bf16n_tensor_op_f32, 256x16x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_256,_16,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
// FP8N x FP8T -> FP32 with 256x32x32 MMA-2CTA
|
||||
TEST(SM100Only_Device_Gemm_e4m3n_e4m3t_void_bf16n_tensor_op_f32, 256x32x128_4x1x1_2sm) {
|
||||
// Tile and cluster shapes
|
||||
// Collective MMA takes tile shape of the MMA operation as input
|
||||
using MmaTileShape_MNK = Shape<_256,_32,_128>;
|
||||
// Cluster size for multicast
|
||||
using ClusterShape_MNK = Shape<_4,_1,_1>;
|
||||
|
||||
// Describe A and B tensors
|
||||
using ElementA = cutlass::float_e4m3_t;
|
||||
constexpr int AlignA = 128 / sizeof_bits<ElementA>::value;
|
||||
using GmemLayoutA = cutlass::layout::ColumnMajor;
|
||||
using ElementB = cutlass::float_e4m3_t;
|
||||
constexpr int AlignB = 128 / sizeof_bits<ElementB>::value;
|
||||
using GmemLayoutB = cutlass::layout::RowMajor;
|
||||
|
||||
// Describe C and D tensors
|
||||
using ElementC = void;
|
||||
constexpr int AlignC = 0;
|
||||
using GmemLayoutC = cutlass::layout::ColumnMajor;
|
||||
using ElementD = cutlass::bfloat16_t;
|
||||
constexpr int AlignD = 128 / sizeof_bits<ElementD>::value;
|
||||
using GmemLayoutD = cutlass::layout::ColumnMajor;
|
||||
|
||||
// Mma's accumulator type
|
||||
using ElementAccumulator = float;
|
||||
// Epilogue computation's precision type
|
||||
using ElementCompute = float;
|
||||
|
||||
//
|
||||
// Construct CollectiveEpilogue
|
||||
//
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
cutlass::epilogue::collective::EpilogueTileAuto, // Epilogue subtile shape. Auto will find a suitable tile shape
|
||||
ElementAccumulator, ElementCompute, // Mma instr's accumulator type and compute precision for epilogue
|
||||
ElementC, GmemLayoutC, AlignC, // C tensor description
|
||||
ElementD, GmemLayoutD, AlignD, // D tensor description
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm // Epilogue schedule policy
|
||||
>::CollectiveOp;
|
||||
|
||||
//
|
||||
// Construct CollectiveMainloop
|
||||
//
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, // Arch and Tensorop spec
|
||||
ElementA, GmemLayoutA, AlignA, // A tensor elem type, layout and alignment requirement
|
||||
ElementB, GmemLayoutB, AlignB, // B tensor elem type, layout and alignment requirement
|
||||
ElementAccumulator, // Mma instruction accumulator type
|
||||
MmaTileShape_MNK, ClusterShape_MNK, // Mma instruction tile shape, cluster shape
|
||||
// Epilogue's SMEM usage that needs to be subtracted from overall SMEM capacity
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmSm100 // Kernel schedule policy. Auto or using targeted scheduling policy
|
||||
>::CollectiveOp;
|
||||
|
||||
// Create Gemm Kernel using CollectiveEpilogue and CollectiveMainloop created by the builders
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
// Run tests
|
||||
auto pass = test::gemm::device::TestAll<Gemm>();
|
||||
// Check results
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif
|
||||
169
test/unit/gemm/device/sm103_gemm_f4_f4_f32_tensor_op_f32_1sm.cu
Normal file
169
test/unit/gemm/device/sm103_gemm_f4_f4_f32_tensor_op_f32_1sm.cu
Normal file
@@ -0,0 +1,169 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32, 256x256x768_2x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32, 256x512x768_2x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_64>, // needs 128x128 block for VS16 case
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_64>, // needs 128x128 block for VS16 case
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,133 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32, 512x256x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16, 512x256x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,133 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32, 512x512x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16, 512x512x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_128>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
cutlass::half_t, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
133
test/unit/gemm/device/sm103_gemm_f4_f4_f32_tensor_op_f32_2sm.cu
Normal file
133
test/unit/gemm/device/sm103_gemm_f4_f4_f32_tensor_op_f32_2sm.cu
Normal file
@@ -0,0 +1,133 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32, 256x256x768_2x1x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32, 256x512x768_2x4x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32, 512x256x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_128>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16, 512x256x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_128>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,133 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32, 512x512x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_64>,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16, 512x512x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cute::Shape<cute::_128,cute::_64>, // 128x64 and 128x128 are both workabled for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,178 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_group, 512x256x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32_group, 256x512x768_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_group, 256x128x768_2x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32_group, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_group, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32_group, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16_group, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32_group, 512x512x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16_group, 512x512x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC *, 4,
|
||||
float, LayoutC *, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA *, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB *, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::GroupProblemShape<Shape<int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,178 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_ptr_array, 512x256x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32_ptr_array, 256x512x768_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_ptr_array, 256x128x768_2x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs32_ptr_array, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_vs16_ptr_array, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32_ptr_array, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16_ptr_array, 512x384x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,139 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x_ptr_array.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs32_ptr_array, 512x512x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16_ptr_array, 512x512x768_4x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, LayoutC, 4,
|
||||
float, LayoutC, 4,
|
||||
cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutA, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, LayoutB, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_256, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelPtrArrayTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cutlass::gemm::ArrayProblemShape<Shape<int,int,int,int>>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,173 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_streamk, 256x256x768_2x1x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
cutlass::gemm::StreamKScheduler
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_streamk, 256x512x768_2x4x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_4, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
cutlass::gemm::StreamKScheduler
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_2sm_f32_vs16_streamk, 512x384x768_4x2x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassTensorOp,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
// cute::Shape<cute::_128,cute::_64>, // We need 128x128 block for VS16 case for both 1SM and 2SM kernels
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::TmaWarpSpecialized2Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_256, cute::_192, Int<768>>,
|
||||
cute::Shape<cute::_4, cute::_2, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue,
|
||||
cutlass::gemm::StreamKScheduler
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, false /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
97
test/unit/gemm/device/sm103_gemm_f4_tensor_op_f32_nosmem.cu
Normal file
97
test/unit/gemm/device/sm103_gemm_f4_tensor_op_f32_nosmem.cu
Normal file
@@ -0,0 +1,97 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/arch/mma_sm100.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
|
||||
TEST(SM103_Device_Gemm_e2m1t_e2m1n_f32t_tensorop_1sm_f32_nosmem, 256x128x768_2x1x1) {
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
void, cutlass::layout::RowMajor, 4,
|
||||
float, cutlass::layout::RowMajor, 4,
|
||||
cutlass::epilogue::NoSmemWarpSpecialized1Sm
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm103, cutlass::arch::OpClassBlockScaledTensorOp,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::RowMajor, 32,
|
||||
cute::tuple<cutlass::float_e2m1_t,cutlass::float_ue8m0_t>, cutlass::layout::ColumnMajor, 32,
|
||||
float,
|
||||
cute::Shape<cute::_128, cute::_128, Int<768>>,
|
||||
cute::Shape<cute::_2, cute::_1, cute::_1>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs32Sm103
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using namespace test::gemm::device;
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
auto pass = test::gemm::device::TestSmall<Gemm, true /*force_legacy_epilogue*/>(1.0, 0.0);
|
||||
EXPECT_TRUE(pass);
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
|
||||
@@ -0,0 +1,295 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/universal_vector.h>
|
||||
#include <thrust/generate.h>
|
||||
#include <thrust/random.h>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/dispatch_policy.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "cutlass/epilogue/thread/activation.h"
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/packed_stride.hpp"
|
||||
#include "cutlass/util/reference/host/gett.hpp"
|
||||
#include "cutlass/util/device_memory.h"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
template<cute::GMMA::Major SFAMajor,
|
||||
cute::GMMA::Major SFBMajor,
|
||||
int ScaleGranularityM,
|
||||
int ScaleGranularityN,
|
||||
int ScaleGranularityK,
|
||||
class LayoutA,
|
||||
class LayoutB,
|
||||
class LayoutCD,
|
||||
class MmaTileShape,
|
||||
class ClusterShape>
|
||||
bool groupwise_test(
|
||||
Int<ScaleGranularityM>, Int<ScaleGranularityN>, Int<ScaleGranularityK>,
|
||||
LayoutA, LayoutB, LayoutCD,
|
||||
MmaTileShape, ClusterShape) {
|
||||
|
||||
using ScaleConfig = cutlass::detail::Sm90BlockwiseScaleConfig<ScaleGranularityM, ScaleGranularityN, ScaleGranularityK, SFAMajor, SFBMajor>;
|
||||
using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); // Layout type for SFA matrix operand
|
||||
using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); // Layout type for SFB matrix operand
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
float, float,
|
||||
cutlass::float_e4m3_t, LayoutCD, 16,
|
||||
cutlass::float_e4m3_t, LayoutCD, 16,
|
||||
cutlass::epilogue::TmaWarpSpecializedCooperative
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop =
|
||||
typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::float_e4m3_t, cute::tuple<LayoutA, LayoutSFA>, 16,
|
||||
cutlass::float_e4m3_t, cute::tuple<LayoutB, LayoutSFB>, 16,
|
||||
float,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<sizeof(typename CollectiveEpilogue::SharedStorage)>,
|
||||
cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum
|
||||
>::CollectiveOp;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
cute::Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
using StrideA = typename Gemm::GemmKernel::StrideA;
|
||||
using StrideB = typename Gemm::GemmKernel::StrideB;
|
||||
using StrideC = typename Gemm::GemmKernel::StrideC;
|
||||
using StrideD = typename Gemm::GemmKernel::StrideD;
|
||||
|
||||
/// Initialization
|
||||
StrideA stride_A;
|
||||
StrideB stride_B;
|
||||
StrideC stride_C;
|
||||
StrideD stride_D;
|
||||
// Strides just iterate over scalars and have no zeros
|
||||
LayoutSFA layout_SFA;
|
||||
LayoutSFB layout_SFB;
|
||||
|
||||
int alignment_M = max(max((is_same_v<LayoutA, cutlass::layout::ColumnMajor> ? 16 : 1) ,
|
||||
(SFAMajor == cute::GMMA::Major::MN ? CollectiveMainloop::AlignmentSFA : 1)),
|
||||
(is_same_v<LayoutCD, cutlass::layout::ColumnMajor> ? 16 : 1));
|
||||
|
||||
int alignment_N = max(max((is_same_v<LayoutB, cutlass::layout::RowMajor> ? 16 : 1) ,
|
||||
(SFBMajor == cute::GMMA::Major::MN ? CollectiveMainloop::AlignmentSFB : 1)),
|
||||
(is_same_v<LayoutCD, cutlass::layout::RowMajor> ? 16 : 1));
|
||||
|
||||
int alignment_K = max(max((is_same_v<LayoutA, cutlass::layout::RowMajor> ? 16 : 1) ,
|
||||
(SFAMajor == cute::GMMA::Major::K ? CollectiveMainloop::AlignmentSFA : 1)),
|
||||
max((is_same_v<LayoutB, cutlass::layout::ColumnMajor> ? 16 : 1) ,
|
||||
(SFBMajor == cute::GMMA::Major::K ? CollectiveMainloop::AlignmentSFB : 1)));
|
||||
|
||||
alignment_K = (alignment_K / size<2>(MmaTileShape{}) + 1) * size<2>(MmaTileShape{});
|
||||
|
||||
int M = 1024 + alignment_M;
|
||||
int N = 1024 + alignment_N;
|
||||
int K = 512 + alignment_K;
|
||||
EXPECT_TRUE(M % alignment_M == 0);
|
||||
EXPECT_TRUE(N % alignment_N == 0);
|
||||
EXPECT_TRUE(K % alignment_K == 0);
|
||||
EXPECT_TRUE(K % size<2>(MmaTileShape{}) == 0);
|
||||
|
||||
stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, 1));
|
||||
stride_B = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(N, K, 1));
|
||||
stride_C = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, 1));
|
||||
stride_D = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, 1));
|
||||
|
||||
layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(M, N, K, 1));
|
||||
layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(M, N, K, 1));
|
||||
|
||||
thrust::universal_vector<cutlass::float_e4m3_t> tensor_A(M * K);
|
||||
thrust::universal_vector<float> tensor_SFA(cute::size(cute::filter_zeros(layout_SFA)));
|
||||
thrust::universal_vector<cutlass::float_e4m3_t> tensor_B(N * K);
|
||||
thrust::universal_vector<float> tensor_SFB(cute::size(cute::filter_zeros(layout_SFB)));
|
||||
thrust::universal_vector<cutlass::float_e4m3_t> tensor_C(M * N);
|
||||
thrust::universal_vector<cutlass::float_e4m3_t> tensor_D(M * N);
|
||||
thrust::universal_vector<cutlass::float_e4m3_t> tensor_ref_D(M * N);
|
||||
|
||||
thrust::random::default_random_engine engine(2025);
|
||||
thrust::random::uniform_int_distribution<int> dist(-2, 2);
|
||||
|
||||
std::generate(tensor_A.begin(), tensor_A.end(), [&] () {
|
||||
return static_cast<cutlass::float_e4m3_t>(dist(engine));
|
||||
});
|
||||
std::generate(tensor_SFA.begin(), tensor_SFA.end(), [&] () {
|
||||
return static_cast<float>(dist(engine));
|
||||
});
|
||||
std::generate(tensor_B.begin(), tensor_B.end(), [&] () {
|
||||
return static_cast<cutlass::float_e4m3_t>(dist(engine));
|
||||
});
|
||||
std::generate(tensor_SFB.begin(), tensor_SFB.end(), [&] () {
|
||||
return static_cast<float>(dist(engine));
|
||||
});
|
||||
std::generate(tensor_C.begin(), tensor_C.end(), [&] () {
|
||||
return static_cast<cutlass::float_e4m3_t>(dist(engine));
|
||||
});
|
||||
|
||||
typename Gemm::Arguments arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
{M, N, K, 1},
|
||||
{thrust::raw_pointer_cast(tensor_A.data()), stride_A,
|
||||
thrust::raw_pointer_cast(tensor_B.data()), stride_B,
|
||||
thrust::raw_pointer_cast(tensor_SFA.data()), layout_SFA,
|
||||
thrust::raw_pointer_cast(tensor_SFB.data()), layout_SFB},
|
||||
{
|
||||
{}, // epilogue.thread
|
||||
thrust::raw_pointer_cast(tensor_C.data()), stride_C,
|
||||
thrust::raw_pointer_cast(tensor_D.data()), stride_D
|
||||
}
|
||||
};
|
||||
|
||||
auto &fusion_args = arguments.epilogue.thread;
|
||||
fusion_args.alpha = 1.0f;
|
||||
fusion_args.beta = 1.0f;
|
||||
|
||||
size_t workspace_size = Gemm::get_workspace_size(arguments);
|
||||
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
|
||||
|
||||
Gemm gemm;
|
||||
|
||||
EXPECT_TRUE(gemm.can_implement(arguments) == cutlass::Status::kSuccess);
|
||||
EXPECT_TRUE(gemm.initialize(arguments, workspace.get()) == cutlass::Status::kSuccess);
|
||||
EXPECT_TRUE(gemm.run() == cutlass::Status::kSuccess);
|
||||
EXPECT_TRUE(cudaDeviceSynchronize() == cudaSuccess);
|
||||
|
||||
auto A = cute::make_tensor(thrust::raw_pointer_cast(tensor_A.data()),
|
||||
cute::make_layout(cute::make_shape(M, K, 1), stride_A));
|
||||
auto B = cute::make_tensor(thrust::raw_pointer_cast(tensor_B.data()),
|
||||
cute::make_layout(cute::make_shape(N, K, 1), stride_B));
|
||||
auto C = cute::make_tensor(thrust::raw_pointer_cast(tensor_C.data()),
|
||||
cute::make_layout(cute::make_shape(M, N, 1), stride_C));
|
||||
auto D = cute::make_tensor(thrust::raw_pointer_cast(tensor_ref_D.data()),
|
||||
cute::make_layout(cute::make_shape(M, N, 1), stride_D));
|
||||
auto SFA = cute::make_tensor(thrust::raw_pointer_cast(tensor_SFA.data()), layout_SFA);
|
||||
auto SFB = cute::make_tensor(thrust::raw_pointer_cast(tensor_SFB.data()), layout_SFB);
|
||||
|
||||
cutlass::reference::host::GettBlockScalingMainloopParams<
|
||||
float,
|
||||
decltype(A),
|
||||
decltype(SFA),
|
||||
decltype(B),
|
||||
decltype(SFB)
|
||||
> mainloop_params{A, SFA, B, SFB};
|
||||
|
||||
cutlass::reference::host::GettEpilogueParams<
|
||||
float,
|
||||
float,
|
||||
float,
|
||||
float,
|
||||
decltype(C),
|
||||
decltype(D)
|
||||
> epilogue_params;
|
||||
|
||||
epilogue_params.C = C;
|
||||
epilogue_params.D = D;
|
||||
epilogue_params.alpha = 1.0f;
|
||||
epilogue_params.beta = 1.0f;
|
||||
|
||||
// get reference result
|
||||
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
|
||||
|
||||
// compare_reference
|
||||
bool equal = true;
|
||||
for (size_t i = 0; i < tensor_ref_D.size(); ++i) {
|
||||
equal &= (tensor_ref_D[i] == tensor_D[i]);
|
||||
}
|
||||
return equal;
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_f32_align16_blockwise, 128x128x128_1x1x1_1x128x128_scale) {
|
||||
|
||||
bool passed = groupwise_test<cute::GMMA::Major::MN, cute::GMMA::Major::K>(
|
||||
Int<1>{}, Int<128>{}, Int<128>{},
|
||||
cutlass::layout::RowMajor{}, cutlass::layout::ColumnMajor{},
|
||||
cutlass::layout::RowMajor{},
|
||||
Shape<_128,_128,_128>{},
|
||||
Shape<_1,_1,_1>{});
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_f32_align16_blockwise, 128x128x128_1x1x1_1x1x128_scale) {
|
||||
|
||||
bool passed = groupwise_test<cute::GMMA::Major::MN, cute::GMMA::Major::MN>(
|
||||
Int<1>{}, Int<128>{}, Int<128>{},
|
||||
cutlass::layout::RowMajor{}, cutlass::layout::ColumnMajor{},
|
||||
cutlass::layout::RowMajor{},
|
||||
Shape<_256,_128,_128>{},
|
||||
Shape<_2,_1,_1>{});
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_e4m3t_e4m3n_e4m3t_tensorop_f32_align16_blockwise, 128x128x128_1x1x1_1x128x128_k_maj_k_maj_scale) {
|
||||
|
||||
bool passed = groupwise_test<cute::GMMA::Major::K, cute::GMMA::Major::K>(
|
||||
Int<1>{}, Int<128>{}, Int<128>{},
|
||||
cutlass::layout::RowMajor{}, cutlass::layout::ColumnMajor{},
|
||||
cutlass::layout::RowMajor{},
|
||||
Shape<_128,_128,_128>{},
|
||||
Shape<_1,_1,_1>{});
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
}
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
Reference in New Issue
Block a user