leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

v4.1 release

Browse Source
This commit is contained in:
Junkai-Wu
2025-07-03 08:07:53 -04:00
committed by GitHub
parent b995f93317
commit a1aaf2300a
155 changed files with 18407 additions and 6068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
+24 -23
View File
@@ -559,7 +559,8 @@ void BlockScaledGemmOperationProfiler::update_workspace_(
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
int swizzle_size,
bool is_dynamic_cluster_enabled
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
@@ -573,15 +574,17 @@ void BlockScaledGemmOperationProfiler::update_workspace_(
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
if (is_dynamic_cluster_enabled) {
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
}
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
@@ -598,7 +601,8 @@ void BlockScaledGemmOperationProfiler::update_result_(
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
int swizzle_size,
bool is_dynamic_cluster_enabled
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
@@ -609,12 +613,14 @@ void BlockScaledGemmOperationProfiler::update_result_(
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
if (is_dynamic_cluster_enabled) {
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
}
@@ -1377,9 +1383,8 @@ bool BlockScaledGemmOperationProfiler::profile(
library::BlockScaledGemmDescription const &operation_desc =
static_cast<library::BlockScaledGemmDescription const &>(operation->description());
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
auto cluster_shape = operation_desc.tile_description.cluster_shape;
bool is_dynamic_cluster_enabled = cluster_shape.m() == 0 || cluster_shape.n() == 0 || cluster_shape.k() == 0;
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
@@ -1419,7 +1424,7 @@ bool BlockScaledGemmOperationProfiler::profile(
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(gemm_workspace_, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
update_workspace_(gemm_workspace_, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size, is_dynamic_cluster_enabled);
const auto can_implement = operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
if (can_implement != Status::kSuccess) {
@@ -1447,7 +1452,7 @@ bool BlockScaledGemmOperationProfiler::profile(
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size, is_dynamic_cluster_enabled);
curr_result.status = profile_cutlass_(
curr_result,
@@ -1490,16 +1495,12 @@ bool BlockScaledGemmOperationProfiler::profile(
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
if (is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
@@ -1510,7 +1511,7 @@ bool BlockScaledGemmOperationProfiler::profile(
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
if (is_dynamic_cluster_enabled && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
tools/profiler/src/gemm_operation_profiler.cu
+24 -24
View File
@@ -661,7 +661,8 @@ void GemmOperationProfiler::update_workspace_(
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
int swizzle_size,
bool is_dynamic_cluster_enabled
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
@@ -675,16 +676,17 @@ void GemmOperationProfiler::update_workspace_(
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
if (is_dynamic_cluster_enabled) {
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
}
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
gemm_workspace.configuration.ldc = leading_dim[2];
@@ -699,7 +701,8 @@ void GemmOperationProfiler::update_result_(
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
int swizzle_size,
bool is_dynamic_cluster_enabled
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
@@ -711,12 +714,14 @@ void GemmOperationProfiler::update_result_(
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
if (is_dynamic_cluster_enabled) {
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
}
@@ -1576,9 +1581,8 @@ bool GemmOperationProfiler::profile(
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
auto cluster_shape = operation_desc.tile_description.cluster_shape;
bool is_dynamic_cluster_enabled = cluster_shape.m() == 0 || cluster_shape.n() == 0 || cluster_shape.k() == 0;
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
@@ -1618,7 +1622,7 @@ bool GemmOperationProfiler::profile(
workspace.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(workspace, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
update_workspace_(workspace, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size, is_dynamic_cluster_enabled);
const auto can_implement = operation->can_implement(&workspace.configuration, &workspace.arguments);
if (can_implement != Status::kSuccess) {
@@ -1672,7 +1676,7 @@ bool GemmOperationProfiler::profile(
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size, is_dynamic_cluster_enabled);
curr_result.status = profile_cutlass_(
curr_result,
@@ -1712,17 +1716,13 @@ bool GemmOperationProfiler::profile(
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
if (is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
@@ -1733,7 +1733,7 @@ bool GemmOperationProfiler::profile(
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
if (is_dynamic_cluster_enabled && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
tools/profiler/src/grouped_gemm_operation_profiler.cu
+24 -19
View File
@@ -538,23 +538,33 @@ void GroupedGemmOperationProfiler::GroupedGemmProblem::initialize_result(
library::lexical_cast(beta, operation_desc.gemm.element_epilogue));
}
void GroupedGemmOperationProfiler::update_result_(
void GroupedGemmOperationProfiler::update_workspace_and_result_(
GroupedGemmWorkspace &gemm_workspace,
PerformanceResult &result,
ProblemSpace const &problem_space,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
int swizzle_size,
bool is_dynamic_cluster_enabled
) {
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
if (is_dynamic_cluster_enabled) {
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1605,9 +1615,8 @@ bool GroupedGemmOperationProfiler::profile_cutlass_for_fixed_shape_(
library::GroupedGemmDescription const &operation_desc =
static_cast<library::GroupedGemmDescription const &>(operation->description());
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
auto cluster_shape = operation_desc.tile_description.cluster_shape;
bool is_dynamic_cluster_enabled = cluster_shape.m() == 0 || cluster_shape.n() == 0 || cluster_shape.k() == 0;
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
@@ -1636,19 +1645,15 @@ bool GroupedGemmOperationProfiler::profile_cutlass_for_fixed_shape_(
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
if (is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
@@ -1656,13 +1661,13 @@ bool GroupedGemmOperationProfiler::profile_cutlass_for_fixed_shape_(
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
if (is_dynamic_cluster_enabled && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
PerformanceResult curr_result(result_base);
update_result_(curr_result, problem_space, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
update_workspace_and_result_(gemm_workspace_, curr_result, problem_space, raster_order, preferred_cluster, fallback_cluster, swizzle_size, is_dynamic_cluster_enabled);
curr_result.status = profile_cutlass_(
curr_result,
options,