v4.1 release

This commit is contained in:
Junkai-Wu
2025-07-03 08:07:53 -04:00
committed by GitHub
parent b995f93317
commit a1aaf2300a
155 changed files with 18407 additions and 6068 deletions
+110 -116
View File
@@ -190,13 +190,7 @@ private:
template <typename T>
auto make_iterator(T* ptr) {
using namespace cute;
if constexpr (cute::is_subbyte_v<T>) {
return subbyte_iterator<T>(ptr);
}
else {
return ptr;
}
return cute::recast_ptr<T>(ptr);
}
template<class T>
@@ -286,26 +280,26 @@ bool initialize_tensor(
scope_max = 2;
scope_min = 0;
}
else if (bits_input <= 6) {
scope_max = 2;
scope_min = -2;
}
else if (bits_input <= 8) {
if constexpr (
cute::is_same_v<Element, cutlass::float_ue8m0_t>){
scope_max = 4;
scope_min = 1;
}
else {
scope_max = 1;
scope_min = -1;
}
}
else{
scope_max = 4;
@@ -354,11 +348,11 @@ static constexpr bool is_row_or_col_major(){
// Default MMA input Operands : A , B
//
template<
class ScheduleType_,
class Gemm,
class ScheduleType_,
class Gemm,
class ElementA_ = typename Gemm::GemmKernel::ElementA,
class ElementB_ = typename Gemm::GemmKernel::ElementB,
class Enable = void>
class Enable = void>
struct HostCollectiveMainloop {
// Kernel data types
using ElementA = ElementA_;
@@ -520,7 +514,7 @@ struct HostCollectiveMainloop {
Arguments to_args() {
// Runtime datatype selection
if constexpr (not cute::is_same_v<ElementA, typename Gemm::GemmKernel::ElementA>) {
using ArrayElementA = typename Gemm::GemmKernel::CollectiveMainloop::ArrayElementA;
@@ -531,13 +525,13 @@ struct HostCollectiveMainloop {
};
}
else {
Arguments arguments =
Arguments arguments =
{
tensor_A.device_data(), stride_a, tensor_B.device_data(), stride_b
};
return arguments;
}
}
}
auto to_host_args(ProblemShapeType problem_size) {
@@ -555,19 +549,19 @@ struct HostCollectiveMainloop {
auto B = make_tensor(make_iterator(tensor_B.host_data()),
make_layout(make_shape(N, K, L), stride_b));
auto dummy_SFA = cute::make_tensor(static_cast<ElementA*>(nullptr),
cute::make_layout(cute::make_shape(M, K, L), stride_a));
auto dummy_SFB = cute::make_tensor(static_cast<ElementB*>(nullptr),
cute::make_layout(cute::make_shape(N, K, L), stride_b));
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
decltype(B)
, decltype(dummy_SFA),
decltype(dummy_SFB)
> mainloop_params{};
mainloop_params.A = A;
@@ -631,7 +625,7 @@ template<
class ElementB_>
struct HostCollectiveMainloopSparse
{
// Kernel data types
using ElementA = ElementA_;
// CuTe layout A for the kernel's sparse tensorA.
@@ -875,8 +869,8 @@ struct HostCollectiveMainloopSparse
};
template<
class ScheduleType_,
class Gemm,
class ScheduleType_,
class Gemm,
class ElementA_,
class ElementB_
>
@@ -1076,7 +1070,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
tensor_A.resize(a_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(a_coord, stride_factor_A));
tensor_B.resize(b_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(b_coord, stride_factor_B));
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2022));
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2021));
@@ -1098,7 +1092,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
// 2.x host tensor does not natively contain a batch stride or coord, so we spoof if by folding it into the outer mode
auto sfa_coord = cutlass::make_Coord(m_blks * Blk_MN{} * L, k_blks * Blk_SF{});
auto sfb_coord = cutlass::make_Coord(n_blks * Blk_MN{} * L, k_blks * Blk_SF{});
tensor_SFA.resize(sfa_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(sfa_coord, stride_factor_A));
tensor_SFB.resize(sfb_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(sfb_coord, stride_factor_B));
@@ -1145,12 +1139,12 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
make_layout(make_shape(N, K, L), stride_b));
auto SfB = make_tensor(tensor_SFB.host_data(), layout_sfb);
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
decltype(B),
decltype(SfA),
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
decltype(B),
decltype(SfA),
decltype(SfB)
>
>
mainloop_params{A, SfA, B, SfB};
return mainloop_params;
}
@@ -1184,7 +1178,7 @@ template<
class ElementA_,
class ElementB_
>
struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedPingpongBlockScaledSm120<SchedulerPipelineStageCount_>,
struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedPingpongBlockScaledSm120<SchedulerPipelineStageCount_>,
Gemm, ElementA_, ElementB_> : public
HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaledSm100<0,0>,
Gemm, ElementA_, ElementB_> {
@@ -1454,7 +1448,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelSparseTmaWarpSpecializedBlock
// 2.x host tensor does not natively contain a batch stride or coord, so we spoof if by folding it into the outer mode
auto sfa_coord = cutlass::make_Coord(m_blks * Blk_MN{} * L, k_blks * Blk_SF{});
auto sfb_coord = cutlass::make_Coord(n_blks * Blk_MN{} * L, k_blks * Blk_SF{});
tensor_SFA.resize(sfa_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(sfa_coord, stride_factor_A));
tensor_SFB.resize(sfb_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(sfb_coord, stride_factor_B));
@@ -1503,12 +1497,12 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelSparseTmaWarpSpecializedBlock
auto SfB = make_tensor(tensor_SFB.host_data(), layout_sfb);
// return {A, SfA, B, SfB};
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
decltype(B),
decltype(SfA),
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
decltype(A),
decltype(B),
decltype(SfA),
decltype(SfB)
>
>
mainloop_params{A, SfA, B, SfB};
return mainloop_params;
}
@@ -1577,7 +1571,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedCooperative
typename Base::LayoutTagA::Stride stride_factor_A_ = typename Base::LayoutTagA::Stride(),
typename Base::LayoutTagB::Stride stride_factor_B_ = typename Base::LayoutTagB::Stride(),
typename Base::LayoutTagE::Stride stride_factor_E_ = typename Base::LayoutTagE::Stride()
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_,
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_,
stride_factor_B_,
stride_factor_E_) {}
};
@@ -1656,8 +1650,8 @@ struct HostCollectiveDefaultEpilogue {
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): init_C(init_C_), seed(seed_),
stride_factor_C(typename LayoutTagC::Stride()),
): init_C(init_C_), seed(seed_),
stride_factor_C(typename LayoutTagC::Stride()),
stride_factor_D(typename LayoutTagD::Stride()),
check_relative_equality(check_relative_equality_),
use_device_scalars(use_device_scalars_){ }
@@ -1766,7 +1760,7 @@ struct HostCollectiveDefaultEpilogue {
bool passed = equality_check(reference_D.host_view(), tensor_D.host_view());
if(!passed) {
std::cout<<"D is incorrect"<<std::endl;
std::cout<<"D is incorrect"<<std::endl;
}
return passed;
}
@@ -1779,7 +1773,7 @@ struct HostCollectiveDefaultEpilogue {
}
Arguments to_args(ProblemShapeType problem_size) {
Arguments arguments =
Arguments arguments =
{
{alpha, beta},
tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d
@@ -1869,7 +1863,7 @@ struct HostCollectiveEpilogue {
typename Gemm::EpilogueOutputOp>;
static_assert(cute::is_base_of_v<cutlass::epilogue::fusion::FusionOperation, FusionOp>);
// Scale factor Generation related
using SfStrategy = cutlass::reference::host::SfStrategy;
static constexpr bool IsBlockScaleSupported = FusionOp::IsBlockScaleSupported;
@@ -1880,11 +1874,11 @@ struct HostCollectiveEpilogue {
using Sm1xxBlockScaledOutputConfig= cutlass::detail::Sm1xxBlockScaledOutputConfig<SFD_VectorSize,
IsKMajorSFD ? cute::UMMA::Major::K : cute::UMMA::Major::MN>;
using Blk_MN = typename Sm1xxBlockScaledOutputConfig::Blk_MN;
using Blk_SF = typename Sm1xxBlockScaledOutputConfig::Blk_SF;
using Blk_SF = typename Sm1xxBlockScaledOutputConfig::Blk_SF;
using OutputSFAtom = typename Sm1xxBlockScaledOutputConfig::SfAtom;
cutlass::HostTensor<ElementSFD, LayoutTagD> tensor_SFD;
cutlass::HostTensor<ElementSFD, LayoutTagD> reference_SFD;
using ElementCompute = typename FusionOp::ElementCompute;
using ElementScalar = typename FusionOp::ElementScalar;
using ElementBias = non_void_t<typename FusionOp::ElementBias>;
@@ -1968,9 +1962,9 @@ struct HostCollectiveEpilogue {
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): init_scale(init_scale_), init_bias(init_bias_),
init_C(init_C_), seed(seed_),
stride_factor_C(typename LayoutTagC::Stride()),
): init_scale(init_scale_), init_bias(init_bias_),
init_C(init_C_), seed(seed_),
stride_factor_C(typename LayoutTagC::Stride()),
stride_factor_D(typename LayoutTagD::Stride()),
check_relative_equality(check_relative_equality_),
use_device_scalars(use_device_scalars_){ }
@@ -2172,7 +2166,7 @@ struct HostCollectiveEpilogue {
}
}
if constexpr (IsBlockScaleSupported) {
auto m_blks = cutlass::ceil_div(M, cute::size<0>(cute::shape(OutputSFAtom{})));
auto n_blks = cutlass::ceil_div(N, cute::size<1>(cute::shape(OutputSFAtom{})));
@@ -2191,7 +2185,7 @@ struct HostCollectiveEpilogue {
EXPECT_TRUE(initialize_tensor(norm_constant.host_view(), init_scale, seed + 2023));
norm_constant.sync_device();
}
return true;
}
@@ -2258,7 +2252,7 @@ struct HostCollectiveEpilogue {
}
}
#endif
std::cout<<"D is incorrect"<<std::endl;
std::cout<<"D is incorrect"<<std::endl;
}
if constexpr (IsAbsMaxEnabledD) {
@@ -2279,24 +2273,24 @@ struct HostCollectiveEpilogue {
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_Aux.host_view()), 0);
passed &= equality_check(reference_Aux.host_view(), tensor_Aux.host_view());
if(!passed) {
std::cout<<"Aux is incorrect"<<std::endl;
std::cout<<"Aux is incorrect"<<std::endl;
}
if constexpr (IsAbsMaxEnabledAux) {
abs_max_Aux.sync_host();
bool tmp = equality_check(reference_abs_max_Aux.host_view(), abs_max_Aux.host_view());
if(!tmp) {
std::cout<<"AbsMax of Aux is incorrect"<<std::endl;
std::cout<<"AbsMax of Aux is incorrect"<<std::endl;
}
passed &= tmp;
}
}
if constexpr (IsBlockScaleSupported) {
tensor_SFD.sync_host();
bool passed_sf = equality_check(reference_SFD.host_view(), tensor_SFD.host_view());
if(!passed_sf) {
std::cout<<"SF is incorrect"<<std::endl;
std::cout<<"SF is incorrect"<<std::endl;
}
passed &= passed_sf;
}
@@ -2317,7 +2311,7 @@ struct HostCollectiveEpilogue {
file << "\n\nvbeta = \n" << beta.host_view();
} else {
file
<< "\n\nalpha= \n" << alpha.host_view()
<< "\n\nalpha= \n" << alpha.host_view()
<< "\n\nbeta= \n " << beta.host_view();
}
file << "\n\n";
@@ -2360,7 +2354,7 @@ struct HostCollectiveEpilogue {
<< "\n\nReference Aux =\n" << reference_Aux.host_view()
<< "\n\nComputed Aux =\n" << tensor_Aux.host_view();
}
if constexpr (IsBlockScaleSupported) {
file
<< "\n\nSFD Reference =\n" << reference_SFD.host_view()
@@ -2378,7 +2372,7 @@ struct HostCollectiveEpilogue {
auto coord_0 = cutlass::make_Coord(0);
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
auto [M, N, K, L] = problem_shape_MNKL;
Arguments arguments =
Arguments arguments =
{
{},
tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d
@@ -2484,7 +2478,7 @@ struct HostCollectiveEpilogue {
}
}
if constexpr (IsBlockScaleSupported) {
arguments.thread.block_scale_factor_ptr = tensor_SFD.device_data();
arguments.thread.norm_constant_ptr = norm_constant.device_data();
@@ -2550,7 +2544,7 @@ struct HostCollectiveEpilogue {
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, cute::_0{}, cute::_1{})));
}
}();
auto SfD = [&](){
if constexpr (IsBlockScaleSupported) {
auto tensor = make_tensor(detail::make_iterator(reference_SFD.host_data()),
@@ -2574,11 +2568,11 @@ struct HostCollectiveEpilogue {
decltype(Valpha),
decltype(Vbeta),
ActivationFunctor,
decltype(SfD),
Int<SFD_VectorSize>,
decltype(SfD),
Int<SFD_VectorSize>,
cutlass::plus<ElementCompute>,
IsColBiasEnabled
, SfGenStrategy
, SfGenStrategy
> epilogue_params{};
epilogue_params.C = C;
@@ -2593,7 +2587,7 @@ struct HostCollectiveEpilogue {
epilogue_params.scale_d = scale_D.at(coord_0);
}
if constexpr (IsRowBiasEnabled or IsColBiasEnabled or IsDeBiasEnabled)
if constexpr (IsRowBiasEnabled or IsColBiasEnabled or IsDeBiasEnabled)
{
epilogue_params.Bias = Bias;
}
@@ -2628,7 +2622,7 @@ struct HostCollectiveEpilogue {
epilogue_params.Vbeta = Vbeta;
}
}
if constexpr (IsBlockScaleSupported) {
epilogue_params.SfD = SfD;
epilogue_params.st = norm_constant.at(coord_0);
@@ -2643,19 +2637,19 @@ template <
bool force_legacy_epilogue = false,
typename ElementA = typename Gemm::GemmKernel::ElementA,
typename ElementB = typename Gemm::GemmKernel::ElementB
, typename RuntimeDatatypeA = void*
, typename RuntimeDatatypeB = void*
, typename RuntimeDatatypeA = void*
, typename RuntimeDatatypeB = void*
>
struct TestbedImpl {
// Kernel data types
using ScheduleType = typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy::Schedule;
// All Collective MMA operands are defined by HostCollectiveMainloopType based on the schedule type
using HostCollectiveMainloopType = HostCollectiveMainloop<ScheduleType, Gemm, ElementA, ElementB>;
using CollectiveEpilogue = cute::conditional_t<IsDefaultEpilogue<typename Gemm::GemmKernel::CollectiveEpilogue>::value || force_legacy_epilogue,
HostCollectiveDefaultEpilogue<Gemm>,
using CollectiveEpilogue = cute::conditional_t<IsDefaultEpilogue<typename Gemm::GemmKernel::CollectiveEpilogue>::value || force_legacy_epilogue,
HostCollectiveDefaultEpilogue<Gemm>,
HostCollectiveEpilogue<Gemm>>;
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
using ElementAccumulator = typename Gemm::GemmKernel::ElementAccumulator;
using ElementCompute = typename ElementComputeType<Gemm, ElementAccumulator>::Type;
@@ -2666,7 +2660,7 @@ struct TestbedImpl {
using LayoutTagC = typename CollectiveEpilogue::LayoutTagC;
using LayoutTagD = typename CollectiveEpilogue::LayoutTagD;
using InternalElementA = typename Gemm::GemmKernel::ElementA;
using InternalElementB = typename Gemm::GemmKernel::ElementB;
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<InternalElementA>();
@@ -2674,11 +2668,11 @@ struct TestbedImpl {
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<InternalElementB>();
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
uint32_t sm_count;
// Used to force multi-wave tests for persistent kernel schedules
@@ -2705,7 +2699,7 @@ struct TestbedImpl {
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
uint64_t seed_ = kDefaultSeed
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
TestbedImpl(
@@ -2759,7 +2753,7 @@ struct TestbedImpl {
file
<< "problem: " << ' ' << M << "x" << N << "x" << K << ", Batch count = " << L
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
collective_mma_inputs.print_tensors(file);
collective_epilogue.print_tensors(file);
}
@@ -2777,7 +2771,7 @@ struct TestbedImpl {
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
auto mainloop_params = collective_mma_inputs.to_host_args(problem_size);
auto epilogue_params = collective_epilogue.to_host_args(problem_size);
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
bool passed = compare_reference(problem_shape_MNKL, alpha, beta);
@@ -2865,12 +2859,12 @@ struct TestbedImpl {
detail::MaxSwizzleSize max_swizzle = detail::MaxSwizzleSize{},
detail::Splits splits = detail::Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic
, RuntimeDatatypeA runtime_input_datatype_a = {}
, RuntimeDatatypeB runtime_input_datatype_b = {}
, RuntimeDatatypeA runtime_input_datatype_a = {}
, RuntimeDatatypeB runtime_input_datatype_b = {}
)
{
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
CUTLASS_TRACE_HOST("TestbedImpl::run");
CUTLASS_TRACE_HOST("TestbedImpl::run");
#endif
// Fail test if insufficient CUDA device
@@ -2933,12 +2927,12 @@ struct TestbedImpl {
mainloop_args = collective_mma_inputs.to_args();
if constexpr (IsRuntimeDataType) {
mainloop_args.runtime_data_type_a = runtime_input_datatype_a;
mainloop_args.runtime_data_type_b = runtime_input_datatype_b;
}
arguments =
{
@@ -3062,19 +3056,19 @@ template <
bool force_legacy_epilogue = false,
typename ElementA = typename Gemm::GemmKernel::ElementA,
typename ElementB = typename Gemm::GemmKernel::ElementB
, typename RuntimeDatatypeA = void*
, typename RuntimeDatatypeB = void*
, typename RuntimeDatatypeA = void*
, typename RuntimeDatatypeB = void*
>
struct Testbed3x {
using TestBedImpl = typename detail::TestbedImpl<
Gemm,
ActivationFunctor,
force_legacy_epilogue,
ElementA,
Gemm,
ActivationFunctor,
force_legacy_epilogue,
ElementA,
ElementB
, RuntimeDatatypeA
, RuntimeDatatypeB
, RuntimeDatatypeA
, RuntimeDatatypeB
>;
using Kernel = typename Gemm::GemmKernel;
using Epilogue = typename Gemm::GemmKernel::CollectiveEpilogue;
@@ -3115,13 +3109,13 @@ struct Testbed3x {
DecompositionMode decomposition_mode = DecompositionMode::Heuristic,
bool profiling = false,
detail::Iterations iterations = detail::Iterations{}
, RuntimeDatatypeA runtime_input_datatype_a = {}
, RuntimeDatatypeB runtime_input_datatype_b = {}
, RuntimeDatatypeA runtime_input_datatype_a = {}
, RuntimeDatatypeB runtime_input_datatype_b = {}
)
{
return impl_.run(
problem_size, alpha, beta, profiling, iterations, raster_order, max_swizzle, splits, decomposition_mode
, runtime_input_datatype_a, runtime_input_datatype_b
, runtime_input_datatype_a, runtime_input_datatype_b
);
}
};
@@ -3176,7 +3170,7 @@ bool TestGemmPerf3x(int iterations = 20) {
/////////////////////////////////////////////////////////////////////////////////////////////////
//
template <
typename Gemm,
typename Gemm,
typename RuntimeDataTypeA,
typename RuntimeDataTypeB,
bool force_legacy_epilogue = false>
@@ -3266,8 +3260,8 @@ bool TestRuntimeDataTypeSmall(
problem_splits.push_back(detail::Splits{2});
}
for (auto splits : problem_splits) {
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF4Format> &&
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF4Format> &&
cute::is_same_v<RuntimeDataTypeB, cute::UMMA::MXF4Format>) {
// e2m1_e2m1
if (runtime_input_datatype_a == cute::UMMA::MXF4Format::E2M1 &&
@@ -3300,16 +3294,16 @@ bool TestRuntimeDataTypeSmall(
return false;
}
}
else
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF8F6F4Format> &&
else
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF8F6F4Format> &&
cute::is_same_v<RuntimeDataTypeB, cute::UMMA::MXF8F6F4Format>) {
static_assert((cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> ||
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float6_t> ||
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float4_t>) &&
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float4_t>) &&
(cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float8_t> ||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float6_t> ||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>),
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>),
"Runtime datatype must be selected with an appropriate static umbrella data type.");
if constexpr (cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> &&
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>) {
@@ -3483,7 +3477,7 @@ bool TestRuntimeDataTypeSmall(
return false;
}
}
else
else
if constexpr (cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> &&
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float8_t>) {
// e5m2_e5m2
@@ -3622,16 +3616,16 @@ bool TestRuntimeDataTypeSmall(
template <typename Gemm, bool force_legacy_epilogue = false, bool apply_alignment_offset = true, bool test_batched_alpha_beta = false>
bool TestSmall(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::GemmKernel::ElementC, void> ? 0.0 : 1.0,
CheckEquality check_relative_equality = CheckEquality::RELATIVE,
ScalarLoc use_device_scalars = ScalarLoc::ON_DEVICE,
VectorScale vector_scale_mode = VectorScale::ENABLED,
CheckEquality check_relative_equality = CheckEquality::RELATIVE,
ScalarLoc use_device_scalars = ScalarLoc::ON_DEVICE,
VectorScale vector_scale_mode = VectorScale::ENABLED,
std::vector<int> override_problem_size_k = {}) {
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
using ElementScalar = typename Gemm::EpilogueOutputOp::ElementScalar;
using CtaShape_MNK = typename Gemm::GemmKernel::CollectiveMainloop::CtaShape_MNK;
using DispatchPolicy = typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy;
CtaShape_MNK cta_shape;
CtaShape_MNK cta_shape;
Testbed3x<Gemm, cutlass::epilogue::thread::Identity, force_legacy_epilogue> testbed(check_relative_equality, use_device_scalars, vector_scale_mode);
static constexpr int SmCount = 16;
static constexpr int MultiplierOffsetM = 1;
@@ -3901,7 +3895,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
}
catch (std::exception const& e) {
EXPECT_TRUE(false) << "TestAll: testbed.run {"
<< "m: " << m << ", n: " << n << ", k: " << k
<< "m: " << m << ", n: " << n << ", k: " << k
<< ", alpha: " << alpha << ", beta: " << beta
<< ", raster_order: ???"
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)
@@ -3912,7 +3906,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
}
catch (...) {
EXPECT_TRUE(false) << "TestAll: testbed.run {"
<< "m: " << m << ", n: " << n << ", k: " << k
<< "m: " << m << ", n: " << n << ", k: " << k
<< ", alpha: " << alpha << ", beta: " << beta
<< ", raster_order: ???"
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)
@@ -3923,7 +3917,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
}
EXPECT_TRUE(passed) << "TestAll: testbed.run {"
<< "m: " << m << ", n: " << n << ", k: " << k
<< "m: " << m << ", n: " << n << ", k: " << k
<< ", alpha: " << alpha << ", beta: " << beta
<< ", raster_order: ???"
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)