v4.3 tag release update. (#2789)
This commit is contained in:
@@ -39,7 +39,7 @@
|
||||
|
||||
Usage:
|
||||
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_fp4_grouped
|
||||
--m=28672 --n=4 --k=4096 --l=8 --benchmark=benchmark.txt
|
||||
--m=7168 --n=128 --k=512 --group=8 --benchmark=benchmark.txt
|
||||
|
||||
*/
|
||||
|
||||
@@ -112,7 +112,7 @@ struct Options {
|
||||
bool error;
|
||||
bool verification;
|
||||
|
||||
int m, n, k, l;
|
||||
int m, n, k, groups;
|
||||
|
||||
int iterations;
|
||||
|
||||
@@ -122,7 +122,7 @@ struct Options {
|
||||
help(false),
|
||||
error(false),
|
||||
verification(true),
|
||||
m(2048), n(2048), k(2048), l(1),
|
||||
m(2048), n(2048), k(2048), groups(1),
|
||||
iterations(10)
|
||||
{ }
|
||||
|
||||
@@ -138,7 +138,7 @@ struct Options {
|
||||
cmd.get_cmd_line_argument("m", m, 2048);
|
||||
cmd.get_cmd_line_argument("n", n, 2048);
|
||||
cmd.get_cmd_line_argument("k", k, 2048);
|
||||
cmd.get_cmd_line_argument("l", l, 1);
|
||||
cmd.get_cmd_line_argument("groups", groups, 1);
|
||||
cmd.get_cmd_line_argument("iterations", iterations, 10);
|
||||
cmd.get_cmd_line_argument("benchmark", benchmark_path);
|
||||
|
||||
@@ -158,7 +158,7 @@ struct Options {
|
||||
<< " --m=<int> Sets the M extent of the GEMM\n"
|
||||
<< " --n=<int> Sets the N extent of the GEMM\n"
|
||||
<< " --k=<int> Sets the K extent of the GEMM\n"
|
||||
<< " --l=<int> Sets the L extent (batch count) of the GEMM\n"
|
||||
<< " --groups=<int> Sets the groups extent (batch count) of the GEMM\n"
|
||||
<< " --iterations=<int> Set the number of profiling iterations to perform\n"
|
||||
<< " --benchmark=<file> Executes a benchmark problem size\n"
|
||||
<< " --no_verif Do not run verification kernels\n";
|
||||
@@ -283,9 +283,7 @@ struct ExampleRunner {
|
||||
MainloopScheduleType
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShapeGroup = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
using ProblemShapeMax = Shape<int,int,int,int>; // max <M,N,K,L>
|
||||
using ProblemShape = cutlass::gemm::MoEProblemShape<ProblemShapeGroup, ProblemShapeMax>;
|
||||
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
ProblemShape,
|
||||
@@ -340,7 +338,11 @@ struct ExampleRunner {
|
||||
cutlass::HostTensor<ElementSFD, cutlass::layout::PackedVectorLayout> block_reference_SFD;
|
||||
cutlass::HostTensor<ElementCompute, cutlass::layout::PackedVectorLayout> block_Normconst;
|
||||
|
||||
cutlass::DeviceAllocation<typename ProblemShapeGroup::UnderlyingProblemShape> problem_sizes;
|
||||
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
|
||||
|
||||
std::vector<ProblemShape::UnderlyingProblemShape> problem_sizes_host;
|
||||
std::vector<int32_t> tokens_per_expert_host;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
@@ -397,10 +399,11 @@ struct ExampleRunner {
|
||||
// Comparison
|
||||
block_D.sync_host();
|
||||
|
||||
auto [maxM, maxN, maxK, L] = problem_size.max_problem_shape;
|
||||
for (int i = 0; i < problem_size.problem_shape.num_groups; i++) {
|
||||
auto problem = problem_size.problem_shape.get_host_problem_shape(i);
|
||||
auto [maxM, maxN, maxK] = problem_size.get_host_problem_shape(0); //gets max problem shape
|
||||
for (int i = 0; i < problem_size.num_groups; i++) {
|
||||
auto problem = problem_sizes_host.at(i);
|
||||
auto [M, N, K] = problem;
|
||||
printf("group [%d] : M = %d, N = %d, K = %d\n", i, M, N, K);
|
||||
|
||||
// assume all M == maxM
|
||||
auto refD_view = block_reference_D.host_view().subview(cutlass::make_Coord(M * N), cutlass::make_Coord(i * maxN * maxM));
|
||||
@@ -414,7 +417,7 @@ struct ExampleRunner {
|
||||
|
||||
/// Initialize operands to be used in the GEMM and reference GEMM
|
||||
void initialize(ProblemShape const& problem_size) {
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size.max_problem_shape, 1);
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size.get_host_problem_shape(0), problem_size.groups());
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
|
||||
// For SFA and SFB tensors layouts
|
||||
@@ -422,17 +425,11 @@ struct ExampleRunner {
|
||||
// For SFD tensor layout
|
||||
using Sm1xxBlockScaledOutputConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig;
|
||||
|
||||
// printf("\nStrideC = ");
|
||||
// print(StrideC{});
|
||||
|
||||
stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, L});
|
||||
stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, L});
|
||||
stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, L});
|
||||
stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, L});
|
||||
|
||||
// printf("\nstride_C = ");
|
||||
// print(stride_C);
|
||||
|
||||
layout_A = make_layout(make_shape(M, K, L), stride_A);
|
||||
layout_B = make_layout(make_shape(N, K, L), stride_B);
|
||||
layout_C = make_layout(make_shape(M, N, L), stride_C);
|
||||
@@ -441,16 +438,6 @@ struct ExampleRunner {
|
||||
layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, L));
|
||||
layout_SFD = SfdOutputCfg::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, L));
|
||||
|
||||
// printf("\nlayout_A = ");
|
||||
// print(layout_A);
|
||||
// printf("\nlayout_B = ");
|
||||
// print(layout_B);
|
||||
// printf("\nlayout_C = ");
|
||||
// print(layout_C);
|
||||
|
||||
// printf("\nsize(layout_A)=%lld", (long long)size(layout_A));
|
||||
// printf("\n");
|
||||
|
||||
block_A.reset(cutlass::make_Coord(size(layout_A)));
|
||||
block_B.reset(cutlass::make_Coord(size(layout_B)));
|
||||
block_C.reset(cutlass::make_Coord(size(layout_C)));
|
||||
@@ -481,12 +468,12 @@ struct ExampleRunner {
|
||||
}
|
||||
|
||||
/// Load a benchmark
|
||||
std::vector<ProblemShapeGroup::UnderlyingProblemShape> benchmark_problems(std::string const& benchmark_path) {
|
||||
std::vector<ProblemShapeGroup::UnderlyingProblemShape> problem_sizes_host;
|
||||
void benchmark_problems(std::string const& benchmark_path) {
|
||||
|
||||
std::ifstream file(benchmark_path);
|
||||
if (!file.good()) {
|
||||
return {};
|
||||
std::cout << "Issues with benchmark file." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
while (file.good()) {
|
||||
@@ -511,34 +498,68 @@ struct ExampleRunner {
|
||||
problem_sizes_host.push_back({extent.m(), extent.n(), extent.k()});
|
||||
}
|
||||
|
||||
return problem_sizes_host;
|
||||
return;
|
||||
}
|
||||
|
||||
void benchmark_tokens_per_expert(std::string const& benchmark_path) {
|
||||
|
||||
std::ifstream file(benchmark_path);
|
||||
if (!file.good()) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (file.good()) {
|
||||
|
||||
int idx = -1;
|
||||
std::string extent_str;
|
||||
|
||||
file >> idx >> extent_str;
|
||||
|
||||
if (idx < 0 || extent_str.empty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
cutlass::gemm::GemmCoord extent;
|
||||
std::vector<std::string> tokens;
|
||||
|
||||
cutlass::CommandLine::tokenize(tokens, extent_str, 'x');
|
||||
|
||||
for (int i = 0; i < int(tokens.size()); ++i) {
|
||||
extent.at(i) = std::atoi(tokens.at(i).c_str());
|
||||
}
|
||||
tokens_per_expert_host.push_back(extent.n());
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
|
||||
auto problem_sizes_host = benchmark_problems(options.benchmark_path);
|
||||
|
||||
benchmark_problems(options.benchmark_path);
|
||||
if (problem_sizes_host.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
problem_sizes.reset(problem_sizes_host.size());
|
||||
problem_sizes.copy_from_host(problem_sizes_host.data());
|
||||
benchmark_tokens_per_expert(options.benchmark_path);
|
||||
if (tokens_per_expert_host.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ProblemShape problem_size;
|
||||
problem_size.max_problem_shape = ProblemShapeMax{options.m, options.n, options.k, options.l};
|
||||
problem_size.problem_shape.num_groups = problem_sizes_host.size();
|
||||
problem_size.problem_shape.problem_shapes = problem_sizes.get();
|
||||
problem_size.problem_shape.host_problem_shapes = problem_sizes_host.data();
|
||||
tokens_per_expert.reset(tokens_per_expert_host.size());
|
||||
tokens_per_expert.copy_from_host(tokens_per_expert_host.data());
|
||||
|
||||
ProblemShape problem_size {options.m, options.n, options.k, options.groups, tokens_per_expert.get(), tokens_per_expert_host.data()};
|
||||
|
||||
initialize(problem_size);
|
||||
|
||||
typename Gemm::Arguments arguments {
|
||||
typename Gemm::Arguments arguments{
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
problem_size,
|
||||
{ // Mainloop arguments
|
||||
block_A.device_data(), stride_A,
|
||||
block_B.device_data(), stride_B,
|
||||
block_SFA.device_data(), layout_SFA,
|
||||
block_SFB.device_data(), layout_SFB
|
||||
block_A.device_data(),
|
||||
block_B.device_data(),
|
||||
block_SFA.device_data(),
|
||||
block_SFB.device_data()
|
||||
},
|
||||
{ // Epilogue arguments
|
||||
{},
|
||||
@@ -624,7 +645,7 @@ struct ExampleRunner {
|
||||
// Compute average setup and runtime and FLOPs.
|
||||
float elapsed_ms = timer.elapsed_millis();
|
||||
double avg_runtime_ms = double(elapsed_ms) / double(options.iterations);
|
||||
double flops = double(int64_t(2) * options.m * options.n * options.k * options.l) / (avg_runtime_ms / 1000.0);
|
||||
double flops = double(int64_t(2) * options.m * options.n * options.k * options.groups) / (avg_runtime_ms / 1000.0);
|
||||
|
||||
std::cout << " Avg runtime : " << avg_runtime_ms << " ms" << std::endl;
|
||||
std::cout << " TFLOPS : " << flops / 1e12 << std::endl;
|
||||
|
||||
@@ -423,17 +423,11 @@ struct ExampleRunner {
|
||||
// For SFD tensor layout
|
||||
using Sm1xxBlockScaledOutputConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig;
|
||||
|
||||
// printf("\nStrideC = ");
|
||||
// print(StrideC{});
|
||||
|
||||
stride_A = cutlass::make_cute_packed_stride(StrideA{}, {M, K, L});
|
||||
stride_B = cutlass::make_cute_packed_stride(StrideB{}, {N, K, L});
|
||||
stride_C = cutlass::make_cute_packed_stride(StrideC{}, {M, N, L});
|
||||
stride_D = cutlass::make_cute_packed_stride(StrideD{}, {M, N, L});
|
||||
|
||||
// printf("\nstride_C = ");
|
||||
// print(stride_C);
|
||||
|
||||
layout_A = make_layout(make_shape(M, K, L), stride_A);
|
||||
layout_B = make_layout(make_shape(N, K, L), stride_B);
|
||||
layout_C = make_layout(make_shape(M, N, L), stride_C);
|
||||
@@ -442,16 +436,6 @@ struct ExampleRunner {
|
||||
layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, L));
|
||||
layout_SFD = SfdOutputCfg::tile_atom_to_shape_SFD(cute::make_shape(M, N, K, L));
|
||||
|
||||
// printf("\nlayout_A = ");
|
||||
// print(layout_A);
|
||||
// printf("\nlayout_B = ");
|
||||
// print(layout_B);
|
||||
// printf("\nlayout_C = ");
|
||||
// print(layout_C);
|
||||
|
||||
// printf("\nsize(layout_A)=%lld", (long long)size(layout_A));
|
||||
// printf("\n");
|
||||
|
||||
block_A.reset(cutlass::make_Coord(size(layout_A)));
|
||||
block_B.reset(cutlass::make_Coord(size(layout_B)));
|
||||
block_C.reset(cutlass::make_Coord(size(layout_C)));
|
||||
@@ -481,27 +465,53 @@ struct ExampleRunner {
|
||||
block_Normconst.sync_device();
|
||||
}
|
||||
|
||||
/// Populates a Gemm::Arguments structure from the given commandline options
|
||||
typename Gemm::Arguments args_from_options(ProblemShapeType problem_size, cutlass::KernelHardwareInfo const& hw_info)
|
||||
{
|
||||
if constexpr (std::is_same_v<MainloopScheduleType, cutlass::gemm::KernelMixedTmaCpAsyncWarpSpecialized1SmBlockScaledSm100>){
|
||||
return typename Gemm::Arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{ // Mainloop arguments
|
||||
block_A.device_data(),
|
||||
block_B.device_data(),
|
||||
block_SFA.device_data(),
|
||||
block_SFB.device_data()
|
||||
},
|
||||
{ // Epilogue arguments
|
||||
{},
|
||||
block_C.device_data(), stride_C,
|
||||
block_D.device_data(), stride_D
|
||||
},
|
||||
hw_info
|
||||
};
|
||||
}
|
||||
else {
|
||||
return typename Gemm::Arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{ // Mainloop arguments
|
||||
block_A.device_data(), stride_A,
|
||||
block_B.device_data(), stride_B,
|
||||
block_SFA.device_data(), layout_SFA,
|
||||
block_SFB.device_data(), layout_SFB
|
||||
},
|
||||
{ // Epilogue arguments
|
||||
{},
|
||||
block_C.device_data(), stride_C,
|
||||
block_D.device_data(), stride_D
|
||||
},
|
||||
hw_info
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
|
||||
ProblemShapeType problem_size = ProblemShapeType{options.m, options.n, options.k, options.l};
|
||||
|
||||
initialize(problem_size);
|
||||
|
||||
typename Gemm::Arguments arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{ // Mainloop arguments
|
||||
block_A.device_data(), stride_A,
|
||||
block_B.device_data(), stride_B,
|
||||
block_SFA.device_data(), layout_SFA,
|
||||
block_SFB.device_data(), layout_SFB
|
||||
},
|
||||
{ // Epilogue arguments
|
||||
{},
|
||||
block_C.device_data(), stride_C,
|
||||
block_D.device_data(), stride_D
|
||||
},
|
||||
hw_info
|
||||
};
|
||||
typename Gemm::Arguments arguments = args_from_options(problem_size, hw_info);
|
||||
|
||||
if constexpr (IsBlockScaleSupported) {
|
||||
arguments.epilogue.thread.block_scale_factor_ptr = block_SFD.device_data();
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
|
||||
Usage:
|
||||
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_grouped
|
||||
--m=28672 --n=4 --k=4096 --l=8 --benchmark=benchmark.txt
|
||||
--m=7168 --n=128 --k=512 --groups=8 --benchmark=benchmark.txt
|
||||
|
||||
*/
|
||||
|
||||
@@ -82,7 +82,7 @@ struct Options {
|
||||
bool error;
|
||||
bool verification;
|
||||
|
||||
int m, n, k, l;
|
||||
int m, n, k, groups;
|
||||
|
||||
int iterations;
|
||||
|
||||
@@ -92,7 +92,7 @@ struct Options {
|
||||
help(false),
|
||||
error(false),
|
||||
verification(true),
|
||||
m(2048), n(2048), k(2048), l(1),
|
||||
m(2048), n(2048), k(2048), groups(1),
|
||||
iterations(10)
|
||||
{ }
|
||||
|
||||
@@ -108,7 +108,7 @@ struct Options {
|
||||
cmd.get_cmd_line_argument("m", m, 2048);
|
||||
cmd.get_cmd_line_argument("n", n, 2048);
|
||||
cmd.get_cmd_line_argument("k", k, 2048);
|
||||
cmd.get_cmd_line_argument("l", l, 1);
|
||||
cmd.get_cmd_line_argument("groups", groups, 1);
|
||||
cmd.get_cmd_line_argument("iterations", iterations, 10);
|
||||
cmd.get_cmd_line_argument("benchmark", benchmark_path);
|
||||
|
||||
@@ -128,7 +128,7 @@ struct Options {
|
||||
<< " --m=<int> Sets the M extent of the GEMM\n"
|
||||
<< " --n=<int> Sets the N extent of the GEMM\n"
|
||||
<< " --k=<int> Sets the K extent of the GEMM\n"
|
||||
<< " --l=<int> Sets the L extent (batch count) of the GEMM\n"
|
||||
<< " --groups=<int> Sets the groups extent (batch count) of the GEMM\n"
|
||||
<< " --iterations=<int> Set the number of profiling iterations to perform\n"
|
||||
<< " --benchmark=<file> Executes a benchmark problem size\n"
|
||||
<< " --no_verif Do not run verification kernels\n";
|
||||
@@ -219,9 +219,7 @@ struct ExampleRunner {
|
||||
MainloopScheduleType
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShapeGroup = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
using ProblemShapeMax = Shape<int,int,int,int>; // max <M,N,K,L>
|
||||
using ProblemShape = cutlass::gemm::MoEProblemShape<ProblemShapeGroup, ProblemShapeMax>;
|
||||
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
ProblemShape,
|
||||
@@ -232,8 +230,6 @@ struct ExampleRunner {
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
|
||||
// using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
|
||||
using StrideA = typename Gemm::GemmKernel::StrideA;
|
||||
using StrideB = typename Gemm::GemmKernel::StrideB;
|
||||
using StrideC = typename Gemm::GemmKernel::StrideC;
|
||||
@@ -249,8 +245,6 @@ struct ExampleRunner {
|
||||
//
|
||||
|
||||
/// Initialization
|
||||
StrideA stride_A;
|
||||
StrideB stride_B;
|
||||
StrideC stride_C;
|
||||
StrideD stride_D;
|
||||
uint64_t seed = 0;
|
||||
@@ -261,7 +255,10 @@ struct ExampleRunner {
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementD> block_D;
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementD> block_ref_D;
|
||||
|
||||
cutlass::DeviceAllocation<typename ProblemShapeGroup::UnderlyingProblemShape> problem_sizes;
|
||||
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
|
||||
|
||||
std::vector<ProblemShape::UnderlyingProblemShape> problem_sizes_host;
|
||||
std::vector<int32_t> tokens_per_expert_host;
|
||||
|
||||
|
||||
//
|
||||
@@ -269,10 +266,11 @@ struct ExampleRunner {
|
||||
//
|
||||
|
||||
bool verify(ProblemShape const& problem_size, float alpha, float beta) {
|
||||
auto [maxM, maxN, maxK, L] = problem_size.max_problem_shape;
|
||||
for (int i = 0; i < problem_size.problem_shape.num_groups; i++) {
|
||||
auto problem = problem_size.problem_shape.get_host_problem_shape(i);
|
||||
auto [maxM, maxN, maxK] = problem_size.get_host_problem_shape(0); //gets max problem shape
|
||||
for (int i = 0; i < problem_size.num_groups; i++) {
|
||||
auto problem = problem_sizes_host.at(i);
|
||||
auto [M, N, K] = problem;
|
||||
printf("group [%d] : M = %d, N = %d, K = %d\n", i, M, N, K);
|
||||
|
||||
cutlass::TensorRef ref_A(block_A.get() + size_t(1) * i * maxM * maxK, Gemm::LayoutA(maxK));
|
||||
cutlass::TensorRef ref_B(block_B.get() + size_t(1) * i * maxN * maxK, Gemm::LayoutB(maxK));
|
||||
@@ -320,11 +318,9 @@ struct ExampleRunner {
|
||||
|
||||
/// Initialize operands to be used in the GEMM and reference GEMM
|
||||
void initialize(ProblemShape const& problem_size) {
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size.max_problem_shape, 1);
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size.get_host_problem_shape(0), problem_size.groups());
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
|
||||
stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
|
||||
stride_B = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(N, K, L));
|
||||
stride_C = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
|
||||
stride_D = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
|
||||
|
||||
@@ -340,12 +336,12 @@ struct ExampleRunner {
|
||||
}
|
||||
|
||||
/// Load a benchmark
|
||||
std::vector<ProblemShapeGroup::UnderlyingProblemShape> benchmark_problems(std::string const& benchmark_path) {
|
||||
std::vector<ProblemShapeGroup::UnderlyingProblemShape> problem_sizes_host;
|
||||
void benchmark_problems(std::string const& benchmark_path) {
|
||||
|
||||
std::ifstream file(benchmark_path);
|
||||
if (!file.good()) {
|
||||
return {};
|
||||
std::cout << "Issues with benchmark file." << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
while (file.good()) {
|
||||
@@ -370,30 +366,64 @@ struct ExampleRunner {
|
||||
problem_sizes_host.push_back({extent.m(), extent.n(), extent.k()});
|
||||
}
|
||||
|
||||
return problem_sizes_host;
|
||||
return;
|
||||
}
|
||||
|
||||
void benchmark_tokens_per_expert(std::string const& benchmark_path) {
|
||||
|
||||
std::ifstream file(benchmark_path);
|
||||
if (!file.good()) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (file.good()) {
|
||||
|
||||
int idx = -1;
|
||||
std::string extent_str;
|
||||
|
||||
file >> idx >> extent_str;
|
||||
|
||||
if (idx < 0 || extent_str.empty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
cutlass::gemm::GemmCoord extent;
|
||||
std::vector<std::string> tokens;
|
||||
|
||||
cutlass::CommandLine::tokenize(tokens, extent_str, 'x');
|
||||
|
||||
for (int i = 0; i < int(tokens.size()); ++i) {
|
||||
extent.at(i) = std::atoi(tokens.at(i).c_str());
|
||||
}
|
||||
tokens_per_expert_host.push_back(extent.n());
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
|
||||
auto problem_sizes_host = benchmark_problems(options.benchmark_path);
|
||||
|
||||
benchmark_problems(options.benchmark_path);
|
||||
if (problem_sizes_host.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
problem_sizes.reset(problem_sizes_host.size());
|
||||
problem_sizes.copy_from_host(problem_sizes_host.data());
|
||||
benchmark_tokens_per_expert(options.benchmark_path);
|
||||
if (tokens_per_expert_host.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ProblemShape problem_size;
|
||||
problem_size.max_problem_shape = ProblemShapeMax{options.m, options.n, options.k, options.l};
|
||||
problem_size.problem_shape.num_groups = problem_sizes_host.size();
|
||||
problem_size.problem_shape.problem_shapes = problem_sizes.get();
|
||||
problem_size.problem_shape.host_problem_shapes = problem_sizes_host.data();
|
||||
tokens_per_expert.reset(tokens_per_expert_host.size());
|
||||
tokens_per_expert.copy_from_host(tokens_per_expert_host.data());
|
||||
|
||||
ProblemShape problem_size {options.m, options.n, options.k, options.groups, tokens_per_expert.get()};
|
||||
|
||||
initialize(problem_size);
|
||||
|
||||
typename Gemm::Arguments arguments{
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
problem_size,
|
||||
{block_A.get(), stride_A, block_B.get(), stride_B},
|
||||
{block_A.get(), block_B.get()},
|
||||
{{}, // epilogue.thread
|
||||
block_C.get(), stride_C, block_D.get(), stride_D},
|
||||
hw_info
|
||||
@@ -464,7 +494,7 @@ struct ExampleRunner {
|
||||
// Compute average setup and runtime and FLOPs.
|
||||
float elapsed_ms = timer.elapsed_millis();
|
||||
double avg_runtime_ms = double(elapsed_ms) / double(options.iterations);
|
||||
double flops = double(int64_t(2) * options.m * options.n * options.k * options.l) / (avg_runtime_ms / 1000.0);
|
||||
double flops = double(int64_t(2) * options.m * options.n * options.k * options.groups) / (avg_runtime_ms / 1000.0);
|
||||
|
||||
std::cout << " Avg runtime : " << avg_runtime_ms << " ms" << std::endl;
|
||||
std::cout << " TFLOPS : " << flops / 1e12 << std::endl;
|
||||
|
||||
@@ -41,7 +41,7 @@
|
||||
|
||||
To run this example:
|
||||
|
||||
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_rcgrouped --m=2048 --n=2048 --k=2048 --groups=10
|
||||
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_rcgrouped --m=128 --k=128 --groups=10
|
||||
|
||||
The above example command makes all 10 groups to be sized at the given m, n, k sizes.
|
||||
Skipping any of the problem dimensions randomizes it across the different groups.
|
||||
@@ -93,7 +93,8 @@
|
||||
#include "helper.h"
|
||||
using namespace cute;
|
||||
|
||||
using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
using ProblemShape = cutlass::gemm::MoEProblemShape<Shape<int,int,int>>; // <M,N,K> per group
|
||||
|
||||
using ElementA = cutlass::float_e4m3_t; // Element type for A matrix operand
|
||||
using ElementB = cutlass::float_e4m3_t; // Element type for B matrix operand
|
||||
using ElementC = cutlass::half_t; // Element type for C and D matrix operands
|
||||
@@ -192,14 +193,11 @@ using StrideB = typename Gemm::GemmKernel::InternalStrideB;
|
||||
using StrideC = typename Gemm::GemmKernel::InternalStrideC;
|
||||
using StrideD = typename Gemm::GemmKernel::InternalStrideD;
|
||||
|
||||
StrideA stride_A;
|
||||
|
||||
// Host-side allocations
|
||||
std::vector<int64_t> offset_B;
|
||||
std::vector<int64_t> offset_C;
|
||||
std::vector<int64_t> offset_D;
|
||||
|
||||
std::vector<StrideB> stride_B_host;
|
||||
std::vector<StrideC> stride_C_host;
|
||||
std::vector<StrideD> stride_D_host;
|
||||
|
||||
@@ -207,7 +205,7 @@ std::vector<ElementAccumulator> alpha_host;
|
||||
std::vector<ElementAccumulator> beta_host;
|
||||
|
||||
// Device-side allocations
|
||||
cutlass::DeviceAllocation<typename ProblemShape::UnderlyingProblemShape> problem_sizes;
|
||||
cutlass::DeviceAllocation<int32_t> tokens_per_expert;
|
||||
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementA> block_A;
|
||||
cutlass::DeviceAllocation<typename Gemm::ElementB> block_B;
|
||||
@@ -223,7 +221,6 @@ cutlass::DeviceAllocation<const typename Gemm::ElementC *> ptr_C;
|
||||
cutlass::DeviceAllocation<typename Gemm::EpilogueOutputOp::ElementOutput *> ptr_D;
|
||||
cutlass::DeviceAllocation<typename Gemm::EpilogueOutputOp::ElementOutput *> ptr_ref_D;
|
||||
|
||||
cutlass::DeviceAllocation<StrideB> stride_B;
|
||||
cutlass::DeviceAllocation<StrideC> stride_C;
|
||||
cutlass::DeviceAllocation<StrideD> stride_D;
|
||||
|
||||
@@ -251,13 +248,14 @@ struct Options {
|
||||
float beta = FLT_MAX;
|
||||
int iterations = 1000;
|
||||
int warmup = 1000;
|
||||
int m = 1024, n = 2048, k = 512, groups = 10;
|
||||
int m = 128, n = 128, k = 128, groups = 10;
|
||||
double sparse_prob = 0.1;
|
||||
dim3 cluster_shape = dim3(4,2,1);
|
||||
dim3 cluster_shape_fallback = dim3(2,1,1);
|
||||
RasterOrderOptions raster_order = RasterOrderOptions::AlongM;
|
||||
int max_sm_count = INT_MAX;
|
||||
std::string benchmark_path;
|
||||
std::vector<int32_t> tokens_per_expert_host;
|
||||
std::vector<typename ProblemShape::UnderlyingProblemShape> problem_sizes_host;
|
||||
int const tma_alignment_bits = 128;
|
||||
int const alignment = tma_alignment_bits / cutlass::sizeof_bits<ElementA>::value;
|
||||
@@ -342,6 +340,7 @@ struct Options {
|
||||
n = alignment * ((rand() % 64) + 1);
|
||||
}
|
||||
problem_sizes_host.push_back({m, n, k});
|
||||
tokens_per_expert_host.push_back(n);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,8 +511,6 @@ bool initialize_block(
|
||||
scope_min = static_cast<Element>(-8);
|
||||
}
|
||||
|
||||
scope_min = static_cast<Element>(0);
|
||||
scope_max = static_cast<Element>(2);
|
||||
cutlass::reference::device::BlockFillRandomUniform(
|
||||
block.get(), block.size(), seed, scope_max, scope_min, 0);
|
||||
|
||||
@@ -545,7 +542,6 @@ void allocate(const Options &options) {
|
||||
total_elements_C += elements_C;
|
||||
total_elements_D += elements_D;
|
||||
|
||||
stride_B_host.push_back(cutlass::make_cute_packed_stride(StrideB{}, {N, K, 1}));
|
||||
stride_C_host.push_back(cutlass::make_cute_packed_stride(StrideC{}, {M, N, 1}));
|
||||
stride_D_host.push_back(cutlass::make_cute_packed_stride(StrideD{}, {M, N, 1}));
|
||||
|
||||
@@ -558,7 +554,6 @@ void allocate(const Options &options) {
|
||||
block_alpha.reset(options.groups);
|
||||
block_beta.reset(options.groups);
|
||||
|
||||
stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(options.m, options.k, options.groups));
|
||||
auto a_coord = cutlass::make_Coord(options.m * options.groups, options.k);
|
||||
block_A.reset(a_coord.product());
|
||||
|
||||
@@ -569,8 +564,9 @@ void initialize(const Options &options) {
|
||||
|
||||
uint64_t seed = 2020;
|
||||
|
||||
problem_sizes.reset(options.groups);
|
||||
problem_sizes.copy_from_host(options.problem_sizes_host.data());
|
||||
// Setting up tokens_per_expert array
|
||||
tokens_per_expert.reset(options.tokens_per_expert_host.size());
|
||||
tokens_per_expert.copy_from_host(options.tokens_per_expert_host.data());
|
||||
|
||||
//
|
||||
// Assign pointers
|
||||
@@ -601,9 +597,6 @@ void initialize(const Options &options) {
|
||||
ptr_D.reset(options.groups);
|
||||
ptr_D.copy_from_host(ptr_D_host.data());
|
||||
|
||||
stride_B.reset(options.groups);
|
||||
stride_B.copy_from_host(stride_B_host.data());
|
||||
|
||||
stride_C.reset(options.groups);
|
||||
stride_C.copy_from_host(stride_C_host.data());
|
||||
|
||||
@@ -675,12 +668,12 @@ typename Gemm::Arguments args_from_options(Options &options)
|
||||
|
||||
typename Gemm::GemmKernel::TileSchedulerArguments scheduler;
|
||||
scheduler.raster_order = options.raster_order;
|
||||
|
||||
|
||||
arguments = typename Gemm::Arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGrouped,
|
||||
{options.groups, problem_sizes.get(), options.problem_sizes_host.data()},
|
||||
{block_A.get(), stride_A, ptr_B.get(), stride_B.get()},
|
||||
{fusion_args, ptr_C.get(), stride_C.get(), ptr_D.get(), stride_D.get()},
|
||||
{options.m, options.n, options.k, options.groups, tokens_per_expert.get()},
|
||||
{block_A.get(), ptr_B.get()},
|
||||
{fusion_args, ptr_C.get(), nullptr, ptr_D.get(), nullptr},
|
||||
hw_info, scheduler
|
||||
};
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
|
||||
Usage:
|
||||
$ ./examples/92_blackwell_moe_gemm/92_blackwell_moe_gemm_regular
|
||||
--m=28672 --n=4 --k=4096 --l=8
|
||||
--m=7168 --n=128 --k=512 --l=8
|
||||
|
||||
*/
|
||||
|
||||
@@ -320,19 +320,38 @@ struct ExampleRunner {
|
||||
initialize_block(block_C, seed + 2021);
|
||||
}
|
||||
|
||||
|
||||
/// Populates a Gemm::Arguments structure from the given commandline options
|
||||
typename Gemm::Arguments args_from_options(ProblemShapeType problem_size, cutlass::KernelHardwareInfo const& hw_info)
|
||||
{
|
||||
if constexpr (std::is_same_v<MainloopScheduleType, cutlass::gemm::KernelMixedTmaCpAsyncWarpSpecialized1SmSm100>){
|
||||
return typename Gemm::Arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{block_A.get(), block_B.get()},
|
||||
{{}, // epilogue.thread
|
||||
block_C.get(), stride_C, block_D.get(), stride_D},
|
||||
hw_info
|
||||
};
|
||||
}
|
||||
else {
|
||||
return typename Gemm::Arguments {
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{block_A.get(), stride_A, block_B.get(), stride_B},
|
||||
{{}, // epilogue.thread
|
||||
block_C.get(), stride_C, block_D.get(), stride_D},
|
||||
hw_info
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
bool run(Options const& options, cutlass::KernelHardwareInfo const& hw_info) {
|
||||
ProblemShapeType problem_size = ProblemShapeType{options.m, options.n, options.k, options.l};
|
||||
|
||||
initialize(problem_size);
|
||||
|
||||
typename Gemm::Arguments arguments{
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
{block_A.get(), stride_A, block_B.get(), stride_B},
|
||||
{{}, // epilogue.thread
|
||||
block_C.get(), stride_C, block_D.get(), stride_D},
|
||||
hw_info
|
||||
};
|
||||
typename Gemm::Arguments arguments = args_from_options(problem_size, hw_info);
|
||||
|
||||
// arguments.scheduler.max_swizzle_size = options.swizzle;
|
||||
|
||||
@@ -471,7 +490,7 @@ int main(int argc, char const **args) {
|
||||
runner_tma.run(options, hw_info);
|
||||
|
||||
std::cout << "Running kernel with CPASYNC load:" << std::endl;
|
||||
ExampleRunner<cutlass::gemm::KernelWarpSpecialized1SmSm100> runner_cpasync;
|
||||
ExampleRunner runner_cpasync;
|
||||
runner_cpasync.run(options, hw_info);
|
||||
|
||||
std::cout << "Running kernel with mixed TMA+CPASYNC load:" << std::endl;
|
||||
|
||||
@@ -26,18 +26,17 @@
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
set(TEST_MIXTRAL_A --m=28672 --n=4 --k=4096 --l=8)
|
||||
set(TEST_MIXTRAL_B --m=4096 --n=4 --k=14336 --l=8)
|
||||
set(TEST_DEEPSEEK_A --m=4096 --n=1 --k=7168 --l=256)
|
||||
set(TEST_DEEPSEEK_B --m=7168 --n=1 --k=2048 --l=256)
|
||||
set(TEST_IRREGULAR_MNK --m=4080 --n=9 --k=4112 --l=8) # M,N,K not multiples of tile size
|
||||
set(TEST_MIXTRAL_A --m=28672 --n=4 --k=4096 --groups=8 --iterations=0)
|
||||
set(TEST_MIXTRAL_B --m=4096 --n=4 --k=14336 --groups=8 --iterations=0)
|
||||
set(TEST_DEEPSEEK_A --m=4096 --n=1 --k=7168 --groups=256 --iterations=0)
|
||||
set(TEST_DEEPSEEK_B --m=7168 --n=1 --k=2048 --groups=256 --iterations=0)
|
||||
set(TEST_IRREGULAR_MNK --m=4080 --n=9 --k=4112 --groups=8 --iterations=0) # M,N,K not mugroupstipgroupses of tigroupse size
|
||||
|
||||
set(TEST_DEEPSEEK_A_FP4 --m=1024 --n=1 --k=7168 --l=256) # TP=1 shape is too large for PackedVectorLayout
|
||||
set(TEST_DEEPSEEK_B_FP4 --m=7168 --n=1 --k=512 --l=256)
|
||||
set(TEST_IRREGULAR_MNK_FP4 --m=4080 --n=9 --k=4160 --l=8)
|
||||
set(TEST_DEEPSEEK_A_FP4 --m=1024 --n=1 --k=7168 --groups=256 --iterations=0) # TP=1 shape is too groupsarge for PackedVectorLayout
|
||||
set(TEST_DEEPSEEK_B_FP4 --m=7168 --n=1 --k=512 --groups=256 --iterations=0)
|
||||
set(TEST_IRREGULAR_MNK_FP4 --m=4080 --n=9 --k=4160 --groups=8 --iterations=0)
|
||||
|
||||
set(TEST_FIXED --m=2048 --n=5120 --k=8192 --iterations=0) # Fixed problem sizes
|
||||
set(TEST_SPARSE_GROUPS --sparse_test --sparse_prob=0.3 --iterations=0)
|
||||
|
||||
if (CUTLASS_NVCC_ARCHS MATCHES 100a)
|
||||
cutlass_example_add_executable(
|
||||
@@ -61,7 +60,6 @@ cutlass_example_add_executable(
|
||||
92_blackwell_moe_gemm_rcgrouped.cu
|
||||
TEST_COMMAND_OPTIONS
|
||||
TEST_FIXED
|
||||
TEST_SPARSE_GROUPS
|
||||
)
|
||||
|
||||
cutlass_example_add_executable(
|
||||
|
||||
Reference in New Issue
Block a user