v4.3 tag release update. (#2789)

This commit is contained in:
Junkai-Wu
2025-11-20 20:49:44 -05:00
committed by GitHub
parent 406e078b29
commit 8cd5bef43a
225 changed files with 23229 additions and 2813 deletions
@@ -39,7 +39,7 @@
Usage:
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_fp4_grouped
--m=28672 --n=4 --k=4096 --l=8 --benchmark=benchmark.txt
--m=7168 --n=128 --k=512 --group=8 --benchmark=benchmark.txt
*/
@@ -112,7 +112,7 @@ struct Options {
bool error;
bool verification;
int m, n, k, l;
int m, n, k, groups;
int iterations;
@@ -122,7 +122,7 @@ struct Options {
help(false),
error(false),
verification(true),
m(2048), n(2048), k(2048), l(1),
m(2048), n(2048), k(2048), groups(1),
iterations(10)
{ }
@@ -138,7 +138,7 @@ struct Options {
cmd.get_cmd_line_argument("m", m, 2048);
cmd.get_cmd_line_argument("n", n, 2048);
cmd.get_cmd_line_argument("k", k, 2048);
cmd.get_cmd_line_argument("l", l, 1);
cmd.get_cmd_line_argument("groups", groups, 1);
cmd.get_cmd_line_argument("iterations", iterations, 10);
cmd.get_cmd_line_argument("benchmark", benchmark_path);
@@ -158,7 +158,7 @@ struct Options {
<< " --m=<int> Sets the M extent of the GEMM\n"
<< " --n=<int> Sets the N extent of the GEMM\n"
<< " --k=<int> Sets the K extent of the GEMM\n"
<< " --l=<int> Sets the L extent (batch count) of the GEMM\n"
<< " --groups=<int> Sets the groups extent (batch count) of the GEMM\n"
<< " --iterations=<int> Set the number of profiling iterations to perform\n"
<< " --benchmark=<file> Executes a benchmark problem size\n"
<< " --no_verif Do not run verification kernels\n";
@@ -283,9 +283,7 @@ struct ExampleRunner {
MainloopScheduleType
>::CollectiveOp;
using ProblemShapeGroup = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using ProblemShapeMax = Shape<int,int,int,int>; // max <M,N,K,L>
using ProblemShape = cutlass::gemm::MoEProblemShape<ProblemShapeGroup, ProblemShapeMax>;
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
ProblemShape,
@@ -340,7 +338,11 @@ struct ExampleRunner {
cutlass::HostTensor<ElementSFD, cutlass::layout::PackedVectorLayout> block_reference_SFD;
cutlass::HostTensor<ElementCompute, cutlass::layout::PackedVectorLayout> block_Normconst;
cutlass::DeviceAllocation<typename ProblemShapeGroup::UnderlyingProblemShape> problem_sizes;
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
std::vector<ProblemShape::UnderlyingProblemShape> problem_sizes_host;
std::vector<int32_t> tokens_per_expert_host;
//
// Methods
@@ -397,10 +399,11 @@ struct ExampleRunner {
// Comparison
block_D.sync_host();
auto [maxM, maxN, maxK, L] = problem_size.max_problem_shape;
for (int i = 0; i < problem_size.problem_shape.num_groups; i++) {
auto problem = problem_size.problem_shape.get_host_problem_shape(i);
auto [maxM, maxN, maxK] = problem_size.get_host_problem_shape(0); //gets max problem shape
for (int i = 0; i < problem_size.num_groups; i++) {
auto problem = problem_sizes_host.at(i);
auto [M, N, K] = problem;
printf("group [%d] : M = %d, N = %d, K = %d\n", i, M, N, K);
// assume all M == maxM
auto refD_view = block_reference_D.host_view().subview(cutlass::make_Coord(M * N), cutlass::make_Coord(i * maxN * maxM));
@@ -414,7 +417,7 @@ struct ExampleRunner {
/// Initialize operands to be used in the GEMM and reference GEMM
void initialize(ProblemShape const& problem_size) {
auto problem_shape_MNKL = cute::append<4>(problem_size.max_problem_shape, 1);
auto problem_shape_MNKL = cute::append<4>(problem_size.get_host_problem_shape(0), problem_size.groups());
auto [M, N, K, L] = problem_shape_MNKL;
// For SFA and SFB tensors layouts
@@ -422,17 +425,11 @@ struct ExampleRunner {
// For SFD tensor layout
using Sm1xxBlockScaledOutputConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig;
// printf("\nStrideC = ");
// print(StrideC{});
stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, L});
stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, L});
stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, L});
stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, L});
// printf("\nstride_C = ");
// print(stride_C);
layout_A = make_layout(make_shape(M, K, L), stride_A);
layout_B = make_layout(make_shape(N, K, L), stride_B);
layout_C = make_layout(make_shape(M, N, L), stride_C);
@@ -441,16 +438,6 @@ struct ExampleRunner {
layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, L));
layout_SFD = SfdOutputCfg::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, L));
// printf("\nlayout_A = ");
// print(layout_A);
// printf("\nlayout_B = ");
// print(layout_B);
// printf("\nlayout_C = ");
// print(layout_C);
// printf("\nsize(layout_A)=%lld", (long long)size(layout_A));
// printf("\n");
block_A.reset(cutlass::make_Coord(size(layout_A)));
block_B.reset(cutlass::make_Coord(size(layout_B)));
block_C.reset(cutlass::make_Coord(size(layout_C)));
@@ -481,12 +468,12 @@ struct ExampleRunner {
}
/// Load a benchmark
std::vector<ProblemShapeGroup::UnderlyingProblemShape> benchmark_problems(std::string const& benchmark_path) {
std::vector<ProblemShapeGroup::UnderlyingProblemShape> problem_sizes_host;
void benchmark_problems(std::string const& benchmark_path) {
std::ifstream file(benchmark_path);
if (!file.good()) {
return {};
std::cout << "Issues with benchmark file." << std::endl;
return;
}
while (file.good()) {
@@ -511,34 +498,68 @@ struct ExampleRunner {
problem_sizes_host.push_back({extent.m(), extent.n(), extent.k()});
}
return problem_sizes_host;
return;
}
void benchmark_tokens_per_expert(std::string const& benchmark_path) {
std::ifstream file(benchmark_path);
if (!file.good()) {
return;
}
while (file.good()) {
int idx = -1;
std::string extent_str;
file >> idx >> extent_str;
if (idx < 0 || extent_str.empty()) {
break;
}
cutlass::gemm::GemmCoord extent;
std::vector<std::string> tokens;
cutlass::CommandLine::tokenize(tokens, extent_str, 'x');
for (int i = 0; i < int(tokens.size()); ++i) {
extent.at(i) = std::atoi(tokens.at(i).c_str());
}
tokens_per_expert_host.push_back(extent.n());
}
return;
}
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
auto problem_sizes_host = benchmark_problems(options.benchmark_path);
benchmark_problems(options.benchmark_path);
if (problem_sizes_host.empty()) {
return false;
}
problem_sizes.reset(problem_sizes_host.size());
problem_sizes.copy_from_host(problem_sizes_host.data());
benchmark_tokens_per_expert(options.benchmark_path);
if (tokens_per_expert_host.empty()) {
return false;
}
ProblemShape problem_size;
problem_size.max_problem_shape = ProblemShapeMax{options.m, options.n, options.k, options.l};
problem_size.problem_shape.num_groups = problem_sizes_host.size();
problem_size.problem_shape.problem_shapes = problem_sizes.get();
problem_size.problem_shape.host_problem_shapes = problem_sizes_host.data();
tokens_per_expert.reset(tokens_per_expert_host.size());
tokens_per_expert.copy_from_host(tokens_per_expert_host.data());
ProblemShape problem_size {options.m, options.n, options.k, options.groups, tokens_per_expert.get(), tokens_per_expert_host.data()};
initialize(problem_size);
typename Gemm::Arguments arguments {
typename Gemm::Arguments arguments{
cutlass::gemm::GemmUniversalMode::kGrouped,
problem_size,
{ // Mainloop arguments
block_A.device_data(), stride_A,
block_B.device_data(), stride_B,
block_SFA.device_data(), layout_SFA,
block_SFB.device_data(), layout_SFB
block_A.device_data(),
block_B.device_data(),
block_SFA.device_data(),
block_SFB.device_data()
},
{ // Epilogue arguments
{},
@@ -624,7 +645,7 @@ struct ExampleRunner {
// Compute average setup and runtime and FLOPs.
float elapsed_ms = timer.elapsed_millis();
double avg_runtime_ms = double(elapsed_ms) / double(options.iterations);
double flops = double(int64_t(2) * options.m * options.n * options.k * options.l) / (avg_runtime_ms / 1000.0);
double flops = double(int64_t(2) * options.m * options.n * options.k * options.groups) / (avg_runtime_ms / 1000.0);
std::cout << " Avg runtime : " << avg_runtime_ms << " ms" << std::endl;
std::cout << " TFLOPS : " << flops / 1e12 << std::endl;
@@ -423,17 +423,11 @@ struct ExampleRunner {
// For SFD tensor layout
using Sm1xxBlockScaledOutputConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig;
// printf("\nStrideC = ");
// print(StrideC{});
stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, L});
stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, L});
stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, L});
stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, L});
// printf("\nstride_C = ");
// print(stride_C);
layout_A = make_layout(make_shape(M, K, L), stride_A);
layout_B = make_layout(make_shape(N, K, L), stride_B);
layout_C = make_layout(make_shape(M, N, L), stride_C);
@@ -442,16 +436,6 @@ struct ExampleRunner {
layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, L));
layout_SFD = SfdOutputCfg::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, L));
// printf("\nlayout_A = ");
// print(layout_A);
// printf("\nlayout_B = ");
// print(layout_B);
// printf("\nlayout_C = ");
// print(layout_C);
// printf("\nsize(layout_A)=%lld", (long long)size(layout_A));
// printf("\n");
block_A.reset(cutlass::make_Coord(size(layout_A)));
block_B.reset(cutlass::make_Coord(size(layout_B)));
block_C.reset(cutlass::make_Coord(size(layout_C)));
@@ -481,27 +465,53 @@ struct ExampleRunner {
block_Normconst.sync_device();
}
/// Populates a Gemm::Arguments structure from the given commandline options
typename Gemm::Arguments args_from_options(ProblemShapeType problem_size, cutlass::KernelHardwareInfo const& hw_info)
{
if constexpr (std::is_same_v<MainloopScheduleType, cutlass::gemm::KernelMixedTmaCpAsyncWarpSpecialized1SmBlockScaledSm100>){
return typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{ // Mainloop arguments
block_A.device_data(),
block_B.device_data(),
block_SFA.device_data(),
block_SFB.device_data()
},
{ // Epilogue arguments
{},
block_C.device_data(), stride_C,
block_D.device_data(), stride_D
},
hw_info
};
}
else {
return typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{ // Mainloop arguments
block_A.device_data(), stride_A,
block_B.device_data(), stride_B,
block_SFA.device_data(), layout_SFA,
block_SFB.device_data(), layout_SFB
},
{ // Epilogue arguments
{},
block_C.device_data(), stride_C,
block_D.device_data(), stride_D
},
hw_info
};
}
}
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
ProblemShapeType problem_size = ProblemShapeType{options.m, options.n, options.k, options.l};
initialize(problem_size);
typename Gemm::Arguments arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{ // Mainloop arguments
block_A.device_data(), stride_A,
block_B.device_data(), stride_B,
block_SFA.device_data(), layout_SFA,
block_SFB.device_data(), layout_SFB
},
{ // Epilogue arguments
{},
block_C.device_data(), stride_C,
block_D.device_data(), stride_D
},
hw_info
};
typename Gemm::Arguments arguments = args_from_options(problem_size, hw_info);
if constexpr (IsBlockScaleSupported) {
arguments.epilogue.thread.block_scale_factor_ptr = block_SFD.device_data();
@@ -39,7 +39,7 @@
Usage:
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_grouped
--m=28672 --n=4 --k=4096 --l=8 --benchmark=benchmark.txt
--m=7168 --n=128 --k=512 --groups=8 --benchmark=benchmark.txt
*/
@@ -82,7 +82,7 @@ struct Options {
bool error;
bool verification;
int m, n, k, l;
int m, n, k, groups;
int iterations;
@@ -92,7 +92,7 @@ struct Options {
help(false),
error(false),
verification(true),
m(2048), n(2048), k(2048), l(1),
m(2048), n(2048), k(2048), groups(1),
iterations(10)
{ }
@@ -108,7 +108,7 @@ struct Options {
cmd.get_cmd_line_argument("m", m, 2048);
cmd.get_cmd_line_argument("n", n, 2048);
cmd.get_cmd_line_argument("k", k, 2048);
cmd.get_cmd_line_argument("l", l, 1);
cmd.get_cmd_line_argument("groups", groups, 1);
cmd.get_cmd_line_argument("iterations", iterations, 10);
cmd.get_cmd_line_argument("benchmark", benchmark_path);
@@ -128,7 +128,7 @@ struct Options {
<< " --m=<int> Sets the M extent of the GEMM\n"
<< " --n=<int> Sets the N extent of the GEMM\n"
<< " --k=<int> Sets the K extent of the GEMM\n"
<< " --l=<int> Sets the L extent (batch count) of the GEMM\n"
<< " --groups=<int> Sets the groups extent (batch count) of the GEMM\n"
<< " --iterations=<int> Set the number of profiling iterations to perform\n"
<< " --benchmark=<file> Executes a benchmark problem size\n"
<< " --no_verif Do not run verification kernels\n";
@@ -219,9 +219,7 @@ struct ExampleRunner {
MainloopScheduleType
>::CollectiveOp;
using ProblemShapeGroup = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using ProblemShapeMax = Shape<int,int,int,int>; // max <M,N,K,L>
using ProblemShape = cutlass::gemm::MoEProblemShape<ProblemShapeGroup, ProblemShapeMax>;
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
ProblemShape,
@@ -232,8 +230,6 @@ struct ExampleRunner {
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
// using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
using StrideA = typename Gemm::GemmKernel::StrideA;
using StrideB = typename Gemm::GemmKernel::StrideB;
using StrideC = typename Gemm::GemmKernel::StrideC;
@@ -249,8 +245,6 @@ struct ExampleRunner {
//
/// Initialization
StrideA stride_A;
StrideB stride_B;
StrideC stride_C;
StrideD stride_D;
uint64_t seed = 0;
@@ -261,7 +255,10 @@ struct ExampleRunner {
cutlass::DeviceAllocation<typename Gemm::ElementD> block_D;
cutlass::DeviceAllocation<typename Gemm::ElementD> block_ref_D;
cutlass::DeviceAllocation<typename ProblemShapeGroup::UnderlyingProblemShape> problem_sizes;
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
std::vector<ProblemShape::UnderlyingProblemShape> problem_sizes_host;
std::vector<int32_t> tokens_per_expert_host;
//
@@ -269,10 +266,11 @@ struct ExampleRunner {
//
bool verify(ProblemShape const& problem_size, float alpha, float beta) {
auto [maxM, maxN, maxK, L] = problem_size.max_problem_shape;
for (int i = 0; i < problem_size.problem_shape.num_groups; i++) {
auto problem = problem_size.problem_shape.get_host_problem_shape(i);
auto [maxM, maxN, maxK] = problem_size.get_host_problem_shape(0); //gets max problem shape
for (int i = 0; i < problem_size.num_groups; i++) {
auto problem = problem_sizes_host.at(i);
auto [M, N, K] = problem;
printf("group [%d] : M = %d, N = %d, K = %d\n", i, M, N, K);
cutlass::TensorRef ref_A(block_A.get() + size_t(1) * i * maxM * maxK, Gemm::LayoutA(maxK));
cutlass::TensorRef ref_B(block_B.get() + size_t(1) * i * maxN * maxK, Gemm::LayoutB(maxK));
@@ -320,11 +318,9 @@ struct ExampleRunner {
/// Initialize operands to be used in the GEMM and reference GEMM
void initialize(ProblemShape const& problem_size) {
auto problem_shape_MNKL = cute::append<4>(problem_size.max_problem_shape, 1);
auto problem_shape_MNKL = cute::append<4>(problem_size.get_host_problem_shape(0), problem_size.groups());
auto [M, N, K, L] = problem_shape_MNKL;
stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
stride_B = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(N, K, L));
stride_C = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
stride_D = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
@@ -340,12 +336,12 @@ struct ExampleRunner {
}
/// Load a benchmark
std::vector<ProblemShapeGroup::UnderlyingProblemShape> benchmark_problems(std::string const& benchmark_path) {
std::vector<ProblemShapeGroup::UnderlyingProblemShape> problem_sizes_host;
void benchmark_problems(std::string const& benchmark_path) {
std::ifstream file(benchmark_path);
if (!file.good()) {
return {};
std::cout << "Issues with benchmark file." << std::endl;
return;
}
while (file.good()) {
@@ -370,30 +366,64 @@ struct ExampleRunner {
problem_sizes_host.push_back({extent.m(), extent.n(), extent.k()});
}
return problem_sizes_host;
return;
}
void benchmark_tokens_per_expert(std::string const& benchmark_path) {
std::ifstream file(benchmark_path);
if (!file.good()) {
return;
}
while (file.good()) {
int idx = -1;
std::string extent_str;
file >> idx >> extent_str;
if (idx < 0 || extent_str.empty()) {
break;
}
cutlass::gemm::GemmCoord extent;
std::vector<std::string> tokens;
cutlass::CommandLine::tokenize(tokens, extent_str, 'x');
for (int i = 0; i < int(tokens.size()); ++i) {
extent.at(i) = std::atoi(tokens.at(i).c_str());
}
tokens_per_expert_host.push_back(extent.n());
}
return;
}
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
auto problem_sizes_host = benchmark_problems(options.benchmark_path);
benchmark_problems(options.benchmark_path);
if (problem_sizes_host.empty()) {
return false;
}
problem_sizes.reset(problem_sizes_host.size());
problem_sizes.copy_from_host(problem_sizes_host.data());
benchmark_tokens_per_expert(options.benchmark_path);
if (tokens_per_expert_host.empty()) {
return false;
}
ProblemShape problem_size;
problem_size.max_problem_shape = ProblemShapeMax{options.m, options.n, options.k, options.l};
problem_size.problem_shape.num_groups = problem_sizes_host.size();
problem_size.problem_shape.problem_shapes = problem_sizes.get();
problem_size.problem_shape.host_problem_shapes = problem_sizes_host.data();
tokens_per_expert.reset(tokens_per_expert_host.size());
tokens_per_expert.copy_from_host(tokens_per_expert_host.data());
ProblemShape problem_size {options.m, options.n, options.k, options.groups, tokens_per_expert.get()};
initialize(problem_size);
typename Gemm::Arguments arguments{
cutlass::gemm::GemmUniversalMode::kGrouped,
problem_size,
{block_A.get(), stride_A, block_B.get(), stride_B},
{block_A.get(), block_B.get()},
{{}, // epilogue.thread
block_C.get(), stride_C, block_D.get(), stride_D},
hw_info
@@ -464,7 +494,7 @@ struct ExampleRunner {
// Compute average setup and runtime and FLOPs.
float elapsed_ms = timer.elapsed_millis();
double avg_runtime_ms = double(elapsed_ms) / double(options.iterations);
double flops = double(int64_t(2) * options.m * options.n * options.k * options.l) / (avg_runtime_ms / 1000.0);
double flops = double(int64_t(2) * options.m * options.n * options.k * options.groups) / (avg_runtime_ms / 1000.0);
std::cout << " Avg runtime : " << avg_runtime_ms << " ms" << std::endl;
std::cout << " TFLOPS : " << flops / 1e12 << std::endl;
@@ -41,7 +41,7 @@
To run this example:
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_rcgrouped --m=2048 --n=2048 --k=2048 --groups=10
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_rcgrouped --m=128 --k=128 --groups=10
The above example command makes all 10 groups to be sized at the given m, n, k sizes.
Skipping any of the problem dimensions randomizes it across the different groups.
@@ -93,7 +93,8 @@
#include "helper.h"
using namespace cute;
using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
using ElementA = cutlass::float_e4m3_t; // Element type for A matrix operand
using ElementB = cutlass::float_e4m3_t; // Element type for B matrix operand
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
@@ -192,14 +193,11 @@ using StrideB = typename Gemm::GemmKernel::InternalStrideB;
using StrideC = typename Gemm::GemmKernel::InternalStrideC;
using StrideD = typename Gemm::GemmKernel::InternalStrideD;
StrideA stride_A;
// Host-side allocations
std::vector<int64_t> offset_B;
std::vector<int64_t> offset_C;
std::vector<int64_t> offset_D;
std::vector<StrideB> stride_B_host;
std::vector<StrideC> stride_C_host;
std::vector<StrideD> stride_D_host;
@@ -207,7 +205,7 @@ std::vector<ElementAccumulator> alpha_host;
std::vector<ElementAccumulator> beta_host;
// Device-side allocations
cutlass::DeviceAllocation<typename ProblemShape::UnderlyingProblemShape> problem_sizes;
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
cutlass::DeviceAllocation<typename Gemm::ElementA> block_A;
cutlass::DeviceAllocation<typename Gemm::ElementB> block_B;
@@ -223,7 +221,6 @@ cutlass::DeviceAllocation<const typename Gemm::ElementC *> ptr_C;
cutlass::DeviceAllocation<typename Gemm::EpilogueOutputOp::ElementOutput *> ptr_D;
cutlass::DeviceAllocation<typename Gemm::EpilogueOutputOp::ElementOutput *> ptr_ref_D;
cutlass::DeviceAllocation<StrideB> stride_B;
cutlass::DeviceAllocation<StrideC> stride_C;
cutlass::DeviceAllocation<StrideD> stride_D;
@@ -251,13 +248,14 @@ struct Options {
float beta = FLT_MAX;
int iterations = 1000;
int warmup = 1000;
int m = 1024, n = 2048, k = 512, groups = 10;
int m = 128, n = 128, k = 128, groups = 10;
double sparse_prob = 0.1;
dim3 cluster_shape = dim3(4,2,1);
dim3 cluster_shape_fallback = dim3(2,1,1);
RasterOrderOptions raster_order = RasterOrderOptions::AlongM;
int max_sm_count = INT_MAX;
std::string benchmark_path;
std::vector<int32_t> tokens_per_expert_host;
std::vector<typename ProblemShape::UnderlyingProblemShape> problem_sizes_host;
int const tma_alignment_bits = 128;
int const alignment = tma_alignment_bits / cutlass::sizeof_bits<ElementA>::value;
@@ -342,6 +340,7 @@ struct Options {
n = alignment * ((rand() % 64) + 1);
}
problem_sizes_host.push_back({m, n, k});
tokens_per_expert_host.push_back(n);
}
}
@@ -512,8 +511,6 @@ bool initialize_block(
scope_min = static_cast<Element>(-8);
}
scope_min = static_cast<Element>(0);
scope_max = static_cast<Element>(2);
cutlass::reference::device::BlockFillRandomUniform(
block.get(), block.size(), seed, scope_max, scope_min, 0);
@@ -545,7 +542,6 @@ void allocate(const Options &options) {
total_elements_C += elements_C;
total_elements_D += elements_D;
stride_B_host.push_back(cutlass::make_cute_packed_stride(StrideB{}, {N, K, 1}));
stride_C_host.push_back(cutlass::make_cute_packed_stride(StrideC{}, {M, N, 1}));
stride_D_host.push_back(cutlass::make_cute_packed_stride(StrideD{}, {M, N, 1}));
@@ -558,7 +554,6 @@ void allocate(const Options &options) {
block_alpha.reset(options.groups);
block_beta.reset(options.groups);
stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(options.m, options.k, options.groups));
auto a_coord = cutlass::make_Coord(options.m * options.groups, options.k);
block_A.reset(a_coord.product());
@@ -569,8 +564,9 @@ void initialize(const Options &options) {
uint64_t seed = 2020;
problem_sizes.reset(options.groups);
problem_sizes.copy_from_host(options.problem_sizes_host.data());
// Setting up tokens_per_expert array
tokens_per_expert.reset(options.tokens_per_expert_host.size());
tokens_per_expert.copy_from_host(options.tokens_per_expert_host.data());
//
// Assign pointers
@@ -601,9 +597,6 @@ void initialize(const Options &options) {
ptr_D.reset(options.groups);
ptr_D.copy_from_host(ptr_D_host.data());
stride_B.reset(options.groups);
stride_B.copy_from_host(stride_B_host.data());
stride_C.reset(options.groups);
stride_C.copy_from_host(stride_C_host.data());
@@ -675,12 +668,12 @@ typename Gemm::Arguments args_from_options(Options &options)
typename Gemm::GemmKernel::TileSchedulerArguments scheduler;
scheduler.raster_order = options.raster_order;
arguments = typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGrouped,
{options.groups, problem_sizes.get(), options.problem_sizes_host.data()},
{block_A.get(), stride_A, ptr_B.get(), stride_B.get()},
{fusion_args, ptr_C.get(), stride_C.get(), ptr_D.get(), stride_D.get()},
{options.m, options.n, options.k, options.groups, tokens_per_expert.get()},
{block_A.get(), ptr_B.get()},
{fusion_args, ptr_C.get(), nullptr, ptr_D.get(), nullptr},
hw_info, scheduler
};
@@ -39,7 +39,7 @@
Usage:
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_regular
--m=28672 --n=4 --k=4096 --l=8
--m=7168 --n=128 --k=512 --l=8
*/
@@ -320,19 +320,38 @@ struct ExampleRunner {
initialize_block(block_C, seed + 2021);
}
/// Populates a Gemm::Arguments structure from the given commandline options
typename Gemm::Arguments args_from_options(ProblemShapeType problem_size, cutlass::KernelHardwareInfo const& hw_info)
{
if constexpr (std::is_same_v<MainloopScheduleType, cutlass::gemm::KernelMixedTmaCpAsyncWarpSpecialized1SmSm100>){
return typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{block_A.get(), block_B.get()},
{{}, // epilogue.thread
block_C.get(), stride_C, block_D.get(), stride_D},
hw_info
};
}
else {
return typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{block_A.get(), stride_A, block_B.get(), stride_B},
{{}, // epilogue.thread
block_C.get(), stride_C, block_D.get(), stride_D},
hw_info
};
}
}
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
ProblemShapeType problem_size = ProblemShapeType{options.m, options.n, options.k, options.l};
initialize(problem_size);
typename Gemm::Arguments arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
{block_A.get(), stride_A, block_B.get(), stride_B},
{{}, // epilogue.thread
block_C.get(), stride_C, block_D.get(), stride_D},
hw_info
};
typename Gemm::Arguments arguments = args_from_options(problem_size, hw_info);
// arguments.scheduler.max_swizzle_size = options.swizzle;
@@ -471,7 +490,7 @@ int main(int argc, char const **args) {
runner_tma.run(options, hw_info);
std::cout << "Running kernel with CPASYNC load:" << std::endl;
ExampleRunner<cutlass::gemm::KernelWarpSpecialized1SmSm100> runner_cpasync;
ExampleRunner runner_cpasync;
runner_cpasync.run(options, hw_info);
std::cout << "Running kernel with mixed TMA+CPASYNC load:" << std::endl;
+8 -10
View File
@@ -26,18 +26,17 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(TEST_MIXTRAL_A --m=28672 --n=4 --k=4096 --l=8)
set(TEST_MIXTRAL_B --m=4096 --n=4 --k=14336 --l=8)
set(TEST_DEEPSEEK_A --m=4096 --n=1 --k=7168 --l=256)
set(TEST_DEEPSEEK_B --m=7168 --n=1 --k=2048 --l=256)
set(TEST_IRREGULAR_MNK --m=4080 --n=9 --k=4112 --l=8) # M,N,K not multiples of tile size
set(TEST_MIXTRAL_A --m=28672 --n=4 --k=4096 --groups=8 --iterations=0)
set(TEST_MIXTRAL_B --m=4096 --n=4 --k=14336 --groups=8 --iterations=0)
set(TEST_DEEPSEEK_A --m=4096 --n=1 --k=7168 --groups=256 --iterations=0)
set(TEST_DEEPSEEK_B --m=7168 --n=1 --k=2048 --groups=256 --iterations=0)
set(TEST_IRREGULAR_MNK --m=4080 --n=9 --k=4112 --groups=8 --iterations=0) # M,N,K not mugroupstipgroupses of tigroupse size
set(TEST_DEEPSEEK_A_FP4 --m=1024 --n=1 --k=7168 --l=256) # TP=1 shape is too large for PackedVectorLayout
set(TEST_DEEPSEEK_B_FP4 --m=7168 --n=1 --k=512 --l=256)
set(TEST_IRREGULAR_MNK_FP4 --m=4080 --n=9 --k=4160 --l=8)
set(TEST_DEEPSEEK_A_FP4 --m=1024 --n=1 --k=7168 --groups=256 --iterations=0) # TP=1 shape is too groupsarge for PackedVectorLayout
set(TEST_DEEPSEEK_B_FP4 --m=7168 --n=1 --k=512 --groups=256 --iterations=0)
set(TEST_IRREGULAR_MNK_FP4 --m=4080 --n=9 --k=4160 --groups=8 --iterations=0)
set(TEST_FIXED --m=2048 --n=5120 --k=8192 --iterations=0) # Fixed problem sizes
set(TEST_SPARSE_GROUPS --sparse_test --sparse_prob=0.3 --iterations=0)
if (CUTLASS_NVCC_ARCHS MATCHES 100a)
cutlass_example_add_executable(
@@ -61,7 +60,6 @@ cutlass_example_add_executable(
92_blackwell_moe_gemm_rcgrouped.cu
TEST_COMMAND_OPTIONS
TEST_FIXED
TEST_SPARSE_GROUPS
)
cutlass_example_add_executable(