v4.1 release
This commit is contained in:
@@ -190,13 +190,7 @@ private:
|
||||
|
||||
template <typename T>
|
||||
auto make_iterator(T* ptr) {
|
||||
using namespace cute;
|
||||
if constexpr (cute::is_subbyte_v<T>) {
|
||||
return subbyte_iterator<T>(ptr);
|
||||
}
|
||||
else {
|
||||
return ptr;
|
||||
}
|
||||
return cute::recast_ptr<T>(ptr);
|
||||
}
|
||||
|
||||
template<class T>
|
||||
@@ -286,26 +280,26 @@ bool initialize_tensor(
|
||||
scope_max = 2;
|
||||
scope_min = 0;
|
||||
}
|
||||
|
||||
|
||||
else if (bits_input <= 6) {
|
||||
scope_max = 2;
|
||||
scope_min = -2;
|
||||
}
|
||||
|
||||
|
||||
else if (bits_input <= 8) {
|
||||
|
||||
|
||||
if constexpr (
|
||||
cute::is_same_v<Element, cutlass::float_ue8m0_t>){
|
||||
scope_max = 4;
|
||||
scope_min = 1;
|
||||
}
|
||||
else {
|
||||
|
||||
|
||||
scope_max = 1;
|
||||
scope_min = -1;
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
else{
|
||||
scope_max = 4;
|
||||
@@ -354,11 +348,11 @@ static constexpr bool is_row_or_col_major(){
|
||||
// Default MMA input Operands : A , B
|
||||
//
|
||||
template<
|
||||
class ScheduleType_,
|
||||
class Gemm,
|
||||
class ScheduleType_,
|
||||
class Gemm,
|
||||
class ElementA_ = typename Gemm::GemmKernel::ElementA,
|
||||
class ElementB_ = typename Gemm::GemmKernel::ElementB,
|
||||
class Enable = void>
|
||||
class Enable = void>
|
||||
struct HostCollectiveMainloop {
|
||||
// Kernel data types
|
||||
using ElementA = ElementA_;
|
||||
@@ -520,7 +514,7 @@ struct HostCollectiveMainloop {
|
||||
|
||||
Arguments to_args() {
|
||||
|
||||
|
||||
|
||||
// Runtime datatype selection
|
||||
if constexpr (not cute::is_same_v<ElementA, typename Gemm::GemmKernel::ElementA>) {
|
||||
using ArrayElementA = typename Gemm::GemmKernel::CollectiveMainloop::ArrayElementA;
|
||||
@@ -531,13 +525,13 @@ struct HostCollectiveMainloop {
|
||||
};
|
||||
}
|
||||
else {
|
||||
|
||||
Arguments arguments =
|
||||
|
||||
Arguments arguments =
|
||||
{
|
||||
tensor_A.device_data(), stride_a, tensor_B.device_data(), stride_b
|
||||
};
|
||||
return arguments;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto to_host_args(ProblemShapeType problem_size) {
|
||||
@@ -555,19 +549,19 @@ struct HostCollectiveMainloop {
|
||||
auto B = make_tensor(make_iterator(tensor_B.host_data()),
|
||||
make_layout(make_shape(N, K, L), stride_b));
|
||||
|
||||
|
||||
|
||||
auto dummy_SFA = cute::make_tensor(static_cast<ElementA*>(nullptr),
|
||||
cute::make_layout(cute::make_shape(M, K, L), stride_a));
|
||||
auto dummy_SFB = cute::make_tensor(static_cast<ElementB*>(nullptr),
|
||||
cute::make_layout(cute::make_shape(N, K, L), stride_b));
|
||||
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
decltype(B)
|
||||
|
||||
|
||||
, decltype(dummy_SFA),
|
||||
decltype(dummy_SFB)
|
||||
|
||||
|
||||
> mainloop_params{};
|
||||
|
||||
mainloop_params.A = A;
|
||||
@@ -631,7 +625,7 @@ template<
|
||||
class ElementB_>
|
||||
struct HostCollectiveMainloopSparse
|
||||
{
|
||||
|
||||
|
||||
// Kernel data types
|
||||
using ElementA = ElementA_;
|
||||
// CuTe layout A for the kernel's sparse tensorA.
|
||||
@@ -875,8 +869,8 @@ struct HostCollectiveMainloopSparse
|
||||
};
|
||||
|
||||
template<
|
||||
class ScheduleType_,
|
||||
class Gemm,
|
||||
class ScheduleType_,
|
||||
class Gemm,
|
||||
class ElementA_,
|
||||
class ElementB_
|
||||
>
|
||||
@@ -1076,7 +1070,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
|
||||
|
||||
tensor_A.resize(a_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(a_coord, stride_factor_A));
|
||||
tensor_B.resize(b_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(b_coord, stride_factor_B));
|
||||
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2022));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2021));
|
||||
|
||||
@@ -1098,7 +1092,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
|
||||
// 2.x host tensor does not natively contain a batch stride or coord, so we spoof if by folding it into the outer mode
|
||||
auto sfa_coord = cutlass::make_Coord(m_blks * Blk_MN{} * L, k_blks * Blk_SF{});
|
||||
auto sfb_coord = cutlass::make_Coord(n_blks * Blk_MN{} * L, k_blks * Blk_SF{});
|
||||
|
||||
|
||||
tensor_SFA.resize(sfa_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(sfa_coord, stride_factor_A));
|
||||
tensor_SFB.resize(sfb_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(sfb_coord, stride_factor_B));
|
||||
|
||||
@@ -1145,12 +1139,12 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaled
|
||||
make_layout(make_shape(N, K, L), stride_b));
|
||||
auto SfB = make_tensor(tensor_SFB.host_data(), layout_sfb);
|
||||
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
decltype(B),
|
||||
decltype(SfA),
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
decltype(B),
|
||||
decltype(SfA),
|
||||
decltype(SfB)
|
||||
>
|
||||
>
|
||||
mainloop_params{A, SfA, B, SfB};
|
||||
return mainloop_params;
|
||||
}
|
||||
@@ -1184,7 +1178,7 @@ template<
|
||||
class ElementA_,
|
||||
class ElementB_
|
||||
>
|
||||
struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedPingpongBlockScaledSm120<SchedulerPipelineStageCount_>,
|
||||
struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedPingpongBlockScaledSm120<SchedulerPipelineStageCount_>,
|
||||
Gemm, ElementA_, ElementB_> : public
|
||||
HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedBlockScaledSm100<0,0>,
|
||||
Gemm, ElementA_, ElementB_> {
|
||||
@@ -1454,7 +1448,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelSparseTmaWarpSpecializedBlock
|
||||
// 2.x host tensor does not natively contain a batch stride or coord, so we spoof if by folding it into the outer mode
|
||||
auto sfa_coord = cutlass::make_Coord(m_blks * Blk_MN{} * L, k_blks * Blk_SF{});
|
||||
auto sfb_coord = cutlass::make_Coord(n_blks * Blk_MN{} * L, k_blks * Blk_SF{});
|
||||
|
||||
|
||||
tensor_SFA.resize(sfa_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(sfa_coord, stride_factor_A));
|
||||
tensor_SFB.resize(sfb_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(sfb_coord, stride_factor_B));
|
||||
|
||||
@@ -1503,12 +1497,12 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelSparseTmaWarpSpecializedBlock
|
||||
auto SfB = make_tensor(tensor_SFB.host_data(), layout_sfb);
|
||||
|
||||
// return {A, SfA, B, SfB};
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
decltype(B),
|
||||
decltype(SfA),
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator,
|
||||
decltype(A),
|
||||
decltype(B),
|
||||
decltype(SfA),
|
||||
decltype(SfB)
|
||||
>
|
||||
>
|
||||
mainloop_params{A, SfA, B, SfB};
|
||||
return mainloop_params;
|
||||
}
|
||||
@@ -1577,7 +1571,7 @@ struct HostCollectiveMainloop<cutlass::gemm::KernelTmaWarpSpecializedCooperative
|
||||
typename Base::LayoutTagA::Stride stride_factor_A_ = typename Base::LayoutTagA::Stride(),
|
||||
typename Base::LayoutTagB::Stride stride_factor_B_ = typename Base::LayoutTagB::Stride(),
|
||||
typename Base::LayoutTagE::Stride stride_factor_E_ = typename Base::LayoutTagE::Stride()
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_,
|
||||
) : Base::HostCollectiveMainloop(check_relative_equality_, init_A_, init_B_, seed_, stride_factor_A_,
|
||||
stride_factor_B_,
|
||||
stride_factor_E_) {}
|
||||
};
|
||||
@@ -1656,8 +1650,8 @@ struct HostCollectiveDefaultEpilogue {
|
||||
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = kDefaultSeed
|
||||
): init_C(init_C_), seed(seed_),
|
||||
stride_factor_C(typename LayoutTagC::Stride()),
|
||||
): init_C(init_C_), seed(seed_),
|
||||
stride_factor_C(typename LayoutTagC::Stride()),
|
||||
stride_factor_D(typename LayoutTagD::Stride()),
|
||||
check_relative_equality(check_relative_equality_),
|
||||
use_device_scalars(use_device_scalars_){ }
|
||||
@@ -1766,7 +1760,7 @@ struct HostCollectiveDefaultEpilogue {
|
||||
|
||||
bool passed = equality_check(reference_D.host_view(), tensor_D.host_view());
|
||||
if(!passed) {
|
||||
std::cout<<"D is incorrect"<<std::endl;
|
||||
std::cout<<"D is incorrect"<<std::endl;
|
||||
}
|
||||
return passed;
|
||||
}
|
||||
@@ -1779,7 +1773,7 @@ struct HostCollectiveDefaultEpilogue {
|
||||
}
|
||||
|
||||
Arguments to_args(ProblemShapeType problem_size) {
|
||||
Arguments arguments =
|
||||
Arguments arguments =
|
||||
{
|
||||
{alpha, beta},
|
||||
tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d
|
||||
@@ -1869,7 +1863,7 @@ struct HostCollectiveEpilogue {
|
||||
typename Gemm::EpilogueOutputOp>;
|
||||
static_assert(cute::is_base_of_v<cutlass::epilogue::fusion::FusionOperation, FusionOp>);
|
||||
|
||||
|
||||
|
||||
// Scale factor Generation related
|
||||
using SfStrategy = cutlass::reference::host::SfStrategy;
|
||||
static constexpr bool IsBlockScaleSupported = FusionOp::IsBlockScaleSupported;
|
||||
@@ -1880,11 +1874,11 @@ struct HostCollectiveEpilogue {
|
||||
using Sm1xxBlockScaledOutputConfig= cutlass::detail::Sm1xxBlockScaledOutputConfig<SFD_VectorSize,
|
||||
IsKMajorSFD ? cute::UMMA::Major::K : cute::UMMA::Major::MN>;
|
||||
using Blk_MN = typename Sm1xxBlockScaledOutputConfig::Blk_MN;
|
||||
using Blk_SF = typename Sm1xxBlockScaledOutputConfig::Blk_SF;
|
||||
using Blk_SF = typename Sm1xxBlockScaledOutputConfig::Blk_SF;
|
||||
using OutputSFAtom = typename Sm1xxBlockScaledOutputConfig::SfAtom;
|
||||
cutlass::HostTensor<ElementSFD, LayoutTagD> tensor_SFD;
|
||||
cutlass::HostTensor<ElementSFD, LayoutTagD> reference_SFD;
|
||||
|
||||
|
||||
using ElementCompute = typename FusionOp::ElementCompute;
|
||||
using ElementScalar = typename FusionOp::ElementScalar;
|
||||
using ElementBias = non_void_t<typename FusionOp::ElementBias>;
|
||||
@@ -1968,9 +1962,9 @@ struct HostCollectiveEpilogue {
|
||||
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = kDefaultSeed
|
||||
): init_scale(init_scale_), init_bias(init_bias_),
|
||||
init_C(init_C_), seed(seed_),
|
||||
stride_factor_C(typename LayoutTagC::Stride()),
|
||||
): init_scale(init_scale_), init_bias(init_bias_),
|
||||
init_C(init_C_), seed(seed_),
|
||||
stride_factor_C(typename LayoutTagC::Stride()),
|
||||
stride_factor_D(typename LayoutTagD::Stride()),
|
||||
check_relative_equality(check_relative_equality_),
|
||||
use_device_scalars(use_device_scalars_){ }
|
||||
@@ -2172,7 +2166,7 @@ struct HostCollectiveEpilogue {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
auto m_blks = cutlass::ceil_div(M, cute::size<0>(cute::shape(OutputSFAtom{})));
|
||||
auto n_blks = cutlass::ceil_div(N, cute::size<1>(cute::shape(OutputSFAtom{})));
|
||||
@@ -2191,7 +2185,7 @@ struct HostCollectiveEpilogue {
|
||||
EXPECT_TRUE(initialize_tensor(norm_constant.host_view(), init_scale, seed + 2023));
|
||||
norm_constant.sync_device();
|
||||
}
|
||||
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -2258,7 +2252,7 @@ struct HostCollectiveEpilogue {
|
||||
}
|
||||
}
|
||||
#endif
|
||||
std::cout<<"D is incorrect"<<std::endl;
|
||||
std::cout<<"D is incorrect"<<std::endl;
|
||||
}
|
||||
|
||||
if constexpr (IsAbsMaxEnabledD) {
|
||||
@@ -2279,24 +2273,24 @@ struct HostCollectiveEpilogue {
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_Aux.host_view()), 0);
|
||||
passed &= equality_check(reference_Aux.host_view(), tensor_Aux.host_view());
|
||||
if(!passed) {
|
||||
std::cout<<"Aux is incorrect"<<std::endl;
|
||||
std::cout<<"Aux is incorrect"<<std::endl;
|
||||
}
|
||||
if constexpr (IsAbsMaxEnabledAux) {
|
||||
abs_max_Aux.sync_host();
|
||||
bool tmp = equality_check(reference_abs_max_Aux.host_view(), abs_max_Aux.host_view());
|
||||
if(!tmp) {
|
||||
std::cout<<"AbsMax of Aux is incorrect"<<std::endl;
|
||||
std::cout<<"AbsMax of Aux is incorrect"<<std::endl;
|
||||
}
|
||||
passed &= tmp;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
tensor_SFD.sync_host();
|
||||
bool passed_sf = equality_check(reference_SFD.host_view(), tensor_SFD.host_view());
|
||||
if(!passed_sf) {
|
||||
std::cout<<"SF is incorrect"<<std::endl;
|
||||
std::cout<<"SF is incorrect"<<std::endl;
|
||||
}
|
||||
passed &= passed_sf;
|
||||
}
|
||||
@@ -2317,7 +2311,7 @@ struct HostCollectiveEpilogue {
|
||||
file << "\n\nvbeta = \n" << beta.host_view();
|
||||
} else {
|
||||
file
|
||||
<< "\n\nalpha= \n" << alpha.host_view()
|
||||
<< "\n\nalpha= \n" << alpha.host_view()
|
||||
<< "\n\nbeta= \n " << beta.host_view();
|
||||
}
|
||||
file << "\n\n";
|
||||
@@ -2360,7 +2354,7 @@ struct HostCollectiveEpilogue {
|
||||
<< "\n\nReference Aux =\n" << reference_Aux.host_view()
|
||||
<< "\n\nComputed Aux =\n" << tensor_Aux.host_view();
|
||||
}
|
||||
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
file
|
||||
<< "\n\nSFD Reference =\n" << reference_SFD.host_view()
|
||||
@@ -2378,7 +2372,7 @@ struct HostCollectiveEpilogue {
|
||||
auto coord_0 = cutlass::make_Coord(0);
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
Arguments arguments =
|
||||
Arguments arguments =
|
||||
{
|
||||
{},
|
||||
tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d
|
||||
@@ -2484,7 +2478,7 @@ struct HostCollectiveEpilogue {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
arguments.thread.block_scale_factor_ptr = tensor_SFD.device_data();
|
||||
arguments.thread.norm_constant_ptr = norm_constant.device_data();
|
||||
@@ -2550,7 +2544,7 @@ struct HostCollectiveEpilogue {
|
||||
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, cute::_0{}, cute::_1{})));
|
||||
}
|
||||
}();
|
||||
|
||||
|
||||
auto SfD = [&](){
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
auto tensor = make_tensor(detail::make_iterator(reference_SFD.host_data()),
|
||||
@@ -2574,11 +2568,11 @@ struct HostCollectiveEpilogue {
|
||||
decltype(Valpha),
|
||||
decltype(Vbeta),
|
||||
ActivationFunctor,
|
||||
decltype(SfD),
|
||||
Int<SFD_VectorSize>,
|
||||
decltype(SfD),
|
||||
Int<SFD_VectorSize>,
|
||||
cutlass::plus<ElementCompute>,
|
||||
IsColBiasEnabled
|
||||
, SfGenStrategy
|
||||
, SfGenStrategy
|
||||
> epilogue_params{};
|
||||
|
||||
epilogue_params.C = C;
|
||||
@@ -2593,7 +2587,7 @@ struct HostCollectiveEpilogue {
|
||||
epilogue_params.scale_d = scale_D.at(coord_0);
|
||||
}
|
||||
|
||||
if constexpr (IsRowBiasEnabled or IsColBiasEnabled or IsDeBiasEnabled)
|
||||
if constexpr (IsRowBiasEnabled or IsColBiasEnabled or IsDeBiasEnabled)
|
||||
{
|
||||
epilogue_params.Bias = Bias;
|
||||
}
|
||||
@@ -2628,7 +2622,7 @@ struct HostCollectiveEpilogue {
|
||||
epilogue_params.Vbeta = Vbeta;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
epilogue_params.SfD = SfD;
|
||||
epilogue_params.st = norm_constant.at(coord_0);
|
||||
@@ -2643,19 +2637,19 @@ template <
|
||||
bool force_legacy_epilogue = false,
|
||||
typename ElementA = typename Gemm::GemmKernel::ElementA,
|
||||
typename ElementB = typename Gemm::GemmKernel::ElementB
|
||||
, typename RuntimeDatatypeA = void*
|
||||
, typename RuntimeDatatypeB = void*
|
||||
, typename RuntimeDatatypeA = void*
|
||||
, typename RuntimeDatatypeB = void*
|
||||
>
|
||||
struct TestbedImpl {
|
||||
// Kernel data types
|
||||
using ScheduleType = typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy::Schedule;
|
||||
// All Collective MMA operands are defined by HostCollectiveMainloopType based on the schedule type
|
||||
using HostCollectiveMainloopType = HostCollectiveMainloop<ScheduleType, Gemm, ElementA, ElementB>;
|
||||
|
||||
using CollectiveEpilogue = cute::conditional_t<IsDefaultEpilogue<typename Gemm::GemmKernel::CollectiveEpilogue>::value || force_legacy_epilogue,
|
||||
HostCollectiveDefaultEpilogue<Gemm>,
|
||||
|
||||
using CollectiveEpilogue = cute::conditional_t<IsDefaultEpilogue<typename Gemm::GemmKernel::CollectiveEpilogue>::value || force_legacy_epilogue,
|
||||
HostCollectiveDefaultEpilogue<Gemm>,
|
||||
HostCollectiveEpilogue<Gemm>>;
|
||||
|
||||
|
||||
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
using ElementAccumulator = typename Gemm::GemmKernel::ElementAccumulator;
|
||||
using ElementCompute = typename ElementComputeType<Gemm, ElementAccumulator>::Type;
|
||||
@@ -2666,7 +2660,7 @@ struct TestbedImpl {
|
||||
using LayoutTagC = typename CollectiveEpilogue::LayoutTagC;
|
||||
using LayoutTagD = typename CollectiveEpilogue::LayoutTagD;
|
||||
|
||||
|
||||
|
||||
using InternalElementA = typename Gemm::GemmKernel::ElementA;
|
||||
using InternalElementB = typename Gemm::GemmKernel::ElementB;
|
||||
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<InternalElementA>();
|
||||
@@ -2674,11 +2668,11 @@ struct TestbedImpl {
|
||||
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<InternalElementB>();
|
||||
|
||||
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
|
||||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
|
||||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
|
||||
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
|
||||
|
||||
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
|
||||
|
||||
|
||||
|
||||
uint32_t sm_count;
|
||||
// Used to force multi-wave tests for persistent kernel schedules
|
||||
@@ -2705,7 +2699,7 @@ struct TestbedImpl {
|
||||
cutlass::Distribution::Kind init_scale_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_bias_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = kDefaultSeed
|
||||
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
|
||||
): collective_mma_inputs(HostCollectiveMainloopType(check_relative_equality_, init_A_, init_B_, seed_)),
|
||||
collective_epilogue(CollectiveEpilogue(check_relative_equality_, use_device_scalars_, vector_scale_mode_, init_C_, init_scale_, init_bias_, seed_)) { }
|
||||
|
||||
TestbedImpl(
|
||||
@@ -2759,7 +2753,7 @@ struct TestbedImpl {
|
||||
file
|
||||
<< "problem: " << ' ' << M << "x" << N << "x" << K << ", Batch count = " << L
|
||||
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
|
||||
|
||||
|
||||
collective_mma_inputs.print_tensors(file);
|
||||
collective_epilogue.print_tensors(file);
|
||||
}
|
||||
@@ -2777,7 +2771,7 @@ struct TestbedImpl {
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
|
||||
auto mainloop_params = collective_mma_inputs.to_host_args(problem_size);
|
||||
auto epilogue_params = collective_epilogue.to_host_args(problem_size);
|
||||
|
||||
|
||||
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
|
||||
|
||||
bool passed = compare_reference(problem_shape_MNKL, alpha, beta);
|
||||
@@ -2865,12 +2859,12 @@ struct TestbedImpl {
|
||||
detail::MaxSwizzleSize max_swizzle = detail::MaxSwizzleSize{},
|
||||
detail::Splits splits = detail::Splits{},
|
||||
DecompositionMode decomposition_mode = DecompositionMode::Heuristic
|
||||
, RuntimeDatatypeA runtime_input_datatype_a = {}
|
||||
, RuntimeDatatypeB runtime_input_datatype_b = {}
|
||||
, RuntimeDatatypeA runtime_input_datatype_a = {}
|
||||
, RuntimeDatatypeB runtime_input_datatype_b = {}
|
||||
)
|
||||
{
|
||||
#if (CUTLASS_DEBUG_TRACE_LEVEL > 1)
|
||||
CUTLASS_TRACE_HOST("TestbedImpl::run");
|
||||
CUTLASS_TRACE_HOST("TestbedImpl::run");
|
||||
#endif
|
||||
|
||||
// Fail test if insufficient CUDA device
|
||||
@@ -2933,12 +2927,12 @@ struct TestbedImpl {
|
||||
|
||||
mainloop_args = collective_mma_inputs.to_args();
|
||||
|
||||
|
||||
|
||||
if constexpr (IsRuntimeDataType) {
|
||||
mainloop_args.runtime_data_type_a = runtime_input_datatype_a;
|
||||
mainloop_args.runtime_data_type_b = runtime_input_datatype_b;
|
||||
}
|
||||
|
||||
|
||||
|
||||
arguments =
|
||||
{
|
||||
@@ -3062,19 +3056,19 @@ template <
|
||||
bool force_legacy_epilogue = false,
|
||||
typename ElementA = typename Gemm::GemmKernel::ElementA,
|
||||
typename ElementB = typename Gemm::GemmKernel::ElementB
|
||||
, typename RuntimeDatatypeA = void*
|
||||
, typename RuntimeDatatypeB = void*
|
||||
, typename RuntimeDatatypeA = void*
|
||||
, typename RuntimeDatatypeB = void*
|
||||
>
|
||||
struct Testbed3x {
|
||||
|
||||
using TestBedImpl = typename detail::TestbedImpl<
|
||||
Gemm,
|
||||
ActivationFunctor,
|
||||
force_legacy_epilogue,
|
||||
ElementA,
|
||||
Gemm,
|
||||
ActivationFunctor,
|
||||
force_legacy_epilogue,
|
||||
ElementA,
|
||||
ElementB
|
||||
, RuntimeDatatypeA
|
||||
, RuntimeDatatypeB
|
||||
, RuntimeDatatypeA
|
||||
, RuntimeDatatypeB
|
||||
>;
|
||||
using Kernel = typename Gemm::GemmKernel;
|
||||
using Epilogue = typename Gemm::GemmKernel::CollectiveEpilogue;
|
||||
@@ -3115,13 +3109,13 @@ struct Testbed3x {
|
||||
DecompositionMode decomposition_mode = DecompositionMode::Heuristic,
|
||||
bool profiling = false,
|
||||
detail::Iterations iterations = detail::Iterations{}
|
||||
, RuntimeDatatypeA runtime_input_datatype_a = {}
|
||||
, RuntimeDatatypeB runtime_input_datatype_b = {}
|
||||
, RuntimeDatatypeA runtime_input_datatype_a = {}
|
||||
, RuntimeDatatypeB runtime_input_datatype_b = {}
|
||||
)
|
||||
{
|
||||
return impl_.run(
|
||||
problem_size, alpha, beta, profiling, iterations, raster_order, max_swizzle, splits, decomposition_mode
|
||||
, runtime_input_datatype_a, runtime_input_datatype_b
|
||||
, runtime_input_datatype_a, runtime_input_datatype_b
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -3176,7 +3170,7 @@ bool TestGemmPerf3x(int iterations = 20) {
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
template <
|
||||
typename Gemm,
|
||||
typename Gemm,
|
||||
typename RuntimeDataTypeA,
|
||||
typename RuntimeDataTypeB,
|
||||
bool force_legacy_epilogue = false>
|
||||
@@ -3266,8 +3260,8 @@ bool TestRuntimeDataTypeSmall(
|
||||
problem_splits.push_back(detail::Splits{2});
|
||||
}
|
||||
for (auto splits : problem_splits) {
|
||||
|
||||
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF4Format> &&
|
||||
|
||||
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF4Format> &&
|
||||
cute::is_same_v<RuntimeDataTypeB, cute::UMMA::MXF4Format>) {
|
||||
// e2m1_e2m1
|
||||
if (runtime_input_datatype_a == cute::UMMA::MXF4Format::E2M1 &&
|
||||
@@ -3300,16 +3294,16 @@ bool TestRuntimeDataTypeSmall(
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
else
|
||||
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF8F6F4Format> &&
|
||||
|
||||
else
|
||||
if constexpr (cute::is_same_v<RuntimeDataTypeA, cute::UMMA::MXF8F6F4Format> &&
|
||||
cute::is_same_v<RuntimeDataTypeB, cute::UMMA::MXF8F6F4Format>) {
|
||||
static_assert((cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> ||
|
||||
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float6_t> ||
|
||||
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float4_t>) &&
|
||||
cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float4_t>) &&
|
||||
(cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float8_t> ||
|
||||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float6_t> ||
|
||||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>),
|
||||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>),
|
||||
"Runtime datatype must be selected with an appropriate static umbrella data type.");
|
||||
if constexpr (cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> &&
|
||||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float4_t>) {
|
||||
@@ -3483,7 +3477,7 @@ bool TestRuntimeDataTypeSmall(
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
else
|
||||
if constexpr (cute::is_same_v<InternalElementA, cutlass::type_erased_dynamic_float8_t> &&
|
||||
cute::is_same_v<InternalElementB, cutlass::type_erased_dynamic_float8_t>) {
|
||||
// e5m2_e5m2
|
||||
@@ -3622,16 +3616,16 @@ bool TestRuntimeDataTypeSmall(
|
||||
|
||||
template <typename Gemm, bool force_legacy_epilogue = false, bool apply_alignment_offset = true, bool test_batched_alpha_beta = false>
|
||||
bool TestSmall(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::GemmKernel::ElementC, void> ? 0.0 : 1.0,
|
||||
CheckEquality check_relative_equality = CheckEquality::RELATIVE,
|
||||
ScalarLoc use_device_scalars = ScalarLoc::ON_DEVICE,
|
||||
VectorScale vector_scale_mode = VectorScale::ENABLED,
|
||||
CheckEquality check_relative_equality = CheckEquality::RELATIVE,
|
||||
ScalarLoc use_device_scalars = ScalarLoc::ON_DEVICE,
|
||||
VectorScale vector_scale_mode = VectorScale::ENABLED,
|
||||
std::vector<int> override_problem_size_k = {}) {
|
||||
|
||||
|
||||
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
using ElementScalar = typename Gemm::EpilogueOutputOp::ElementScalar;
|
||||
using CtaShape_MNK = typename Gemm::GemmKernel::CollectiveMainloop::CtaShape_MNK;
|
||||
using DispatchPolicy = typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy;
|
||||
CtaShape_MNK cta_shape;
|
||||
CtaShape_MNK cta_shape;
|
||||
Testbed3x<Gemm, cutlass::epilogue::thread::Identity, force_legacy_epilogue> testbed(check_relative_equality, use_device_scalars, vector_scale_mode);
|
||||
static constexpr int SmCount = 16;
|
||||
static constexpr int MultiplierOffsetM = 1;
|
||||
@@ -3901,7 +3895,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
|
||||
}
|
||||
catch (std::exception const& e) {
|
||||
EXPECT_TRUE(false) << "TestAll: testbed.run {"
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< ", alpha: " << alpha << ", beta: " << beta
|
||||
<< ", raster_order: ???"
|
||||
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)
|
||||
@@ -3912,7 +3906,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
|
||||
}
|
||||
catch (...) {
|
||||
EXPECT_TRUE(false) << "TestAll: testbed.run {"
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< ", alpha: " << alpha << ", beta: " << beta
|
||||
<< ", raster_order: ???"
|
||||
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)
|
||||
@@ -3923,7 +3917,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
|
||||
}
|
||||
|
||||
EXPECT_TRUE(passed) << "TestAll: testbed.run {"
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< "m: " << m << ", n: " << n << ", k: " << k
|
||||
<< ", alpha: " << alpha << ", beta: " << beta
|
||||
<< ", raster_order: ???"
|
||||
<< ", max_swizzle_size: " << static_cast<int>(max_swizzle_size)
|
||||
|
||||
Reference in New Issue
Block a user