leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

v3.9 (#2185)

Browse Source 
* v3.8 update x

* fix blackwell gg

* doc change

* doc change

* doc change

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
This commit is contained in:
Yujia Zhai
2025-03-21 01:52:23 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu Haicheng Wu
parent 8c4d1dc47d
commit 62750a2b75
334 changed files with 91517 additions and 2656 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
+329 -61
View File
@@ -174,7 +174,6 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
@@ -204,7 +203,6 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
// default value
this->cluster_k_fallback = 0;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
// default value
@@ -314,23 +312,61 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
// instantiation for deep profiling
int num_sizes = 8;
this->problem_sizes.resize(num_sizes);
this->leading_dims.resize(num_sizes, {0, 0, 0});
int m0 = 1024;
int n0 = 1024;
int k0 = 1024;
for (int i = 0; i < num_sizes; i++) {
auto m = m0 * (i + 3);
auto n = n0 * (i + 3);
auto k = k0 * (i + 3);
this->problem_sizes[i] = {m, n, k};
this->leading_dims[i] = {
DeviceAllocation::get_packed_layout(operation_desc.A.layout, {int(m), int(k)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.B.layout, {int(k), int(n)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.C.layout, {int(m), int(n)}).front()
};
}
this->raster_orders = {
cutlass::library::RasterOrder::kAlongN,
cutlass::library::RasterOrder::kAlongM
};
this->swizzle_sizes = {1, 2, 4, 8};
this->preferred_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}, {4, 1, 1}, {4, 2, 1}, {4, 4, 1}, {8, 2, 1}
};
this->fallback_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}
};
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScaledGemmDescription const &operation_desc) const {
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_shape.m() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_shape.n() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
}
bytes *= batch_count;
@@ -338,9 +374,16 @@ int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScale
return bytes;
}
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScaledGemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return bytes_with_problem_shape(operation_desc, problem_shape);
}
/// Total number of flops computed
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScaledGemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t flops_ = (int64_t(problem_shape.m()) * problem_shape.n() * problem_shape.k() + problem_shape.m() * problem_shape.n()) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
@@ -362,6 +405,10 @@ int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScale
return flops_;
}
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScaledGemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return flops_with_problem_shape(operation_desc, problem_shape);
}
/// Initializes a performance result
void BlockScaledGemmOperationProfiler::GemmProblem::initialize_result(
@@ -480,8 +527,8 @@ Status BlockScaledGemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.swizzle_size = problem_.swizzle_size;
gemm_workspace_.arguments.raster_order = problem_.raster_order;
gemm_workspace_.arguments.norm_constant = 0;
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.split_k_slices = problem_.split_k_slices;
@@ -499,8 +546,74 @@ Status BlockScaledGemmOperationProfiler::initialize_configuration(
}
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
}
void BlockScaledGemmOperationProfiler::update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
gemm_workspace.arguments.problem_size.n() = problem_shape.n();
gemm_workspace.arguments.problem_size.k() = problem_shape.k();
gemm_workspace.arguments.lda = leading_dim[0];
gemm_workspace.arguments.ldb = leading_dim[1];
gemm_workspace.arguments.ldc = leading_dim[2];
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
gemm_workspace.configuration.ldc = leading_dim[2];
}
void BlockScaledGemmOperationProfiler::update_result_(
PerformanceResult &result,
library::BlockScaledGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
set_argument(result, "m", problem_space, problem_shape.m());
set_argument(result, "n", problem_space, problem_shape.n());
set_argument(result, "k", problem_space, problem_shape.k());
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
/// Initializes the performance result
@@ -619,27 +732,38 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
bool enable_deep_profiling = options.profiling.enable_kernel_performance_search;
int seed_shift = 0;
// When exhaustive performance search (deep profiling) option is enabled, device buffers are initialized to the largest problem shape
// so that later performance search can re-use those buffers.
int init_m = enable_deep_profiling ? std::max(int(problem_.m), problem_.problem_sizes.back().m()) : int(problem_.m);
int init_n = enable_deep_profiling ? std::max(int(problem_.n), problem_.problem_sizes.back().n()) : int(problem_.n);
int init_k = enable_deep_profiling ? std::max(int(problem_.k), problem_.problem_sizes.back().k()) : int(problem_.k);
int init_lda = enable_deep_profiling ? int(std::max(problem_.lda, problem_.leading_dims.back()[0])) : int(problem_.lda);
int init_ldb = enable_deep_profiling ? int(std::max(problem_.ldb, problem_.leading_dims.back()[1])) : int(problem_.ldb);
int init_ldc = enable_deep_profiling ? int(std::max(problem_.ldc, problem_.leading_dims.back()[2])) : int(problem_.ldc);
gemm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
{init_m, init_k},
{init_lda},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
);
int sfa_m = round_up(int(problem_.m), 128);
int sfb_n = round_up(int(problem_.n), 128);
int sfa_sfb_k = round_up(ceil_div(int(problem_.k), operation_desc.SFVecSize), 4);
int sfa_m = round_up(init_m, 128);
int sfb_n = round_up(init_n, 128);
int sfa_sfb_k = round_up(ceil_div(init_k, operation_desc.SFVecSize), 4);
int sfd_m = operation_desc.SFD.layout == cutlass::library::LayoutTypeID::kRowMajor ?
sfa_m : round_up(ceil_div(int(problem_.m), operation_desc.EpilogueSFVecSize), 4);
sfa_m : round_up(ceil_div(init_m, operation_desc.EpilogueSFVecSize), 4);
int sfd_n = operation_desc.SFD.layout == cutlass::library::LayoutTypeID::kRowMajor ?
round_up(ceil_div(int(problem_.n), operation_desc.EpilogueSFVecSize), 4) : sfb_n;
round_up(ceil_div(init_n, operation_desc.EpilogueSFVecSize), 4) : sfb_n;
gemm_workspace_.SFA = device_context.allocate_and_initialize_tensor(
@@ -671,8 +795,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
{init_k, init_n},
{init_ldb},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
@@ -683,8 +807,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"C",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
@@ -695,8 +819,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
@@ -706,8 +830,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
@@ -766,8 +890,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_.arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_.arguments.batch_count = problem_.batch_count;
gemm_workspace_.arguments.lda = problem_.lda;
@@ -1247,42 +1371,186 @@ bool BlockScaledGemmOperationProfiler::profile(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
if (options.profiling.enable_kernel_performance_search || options.profiling.enable_best_kernel_for_fixed_shape) {
library::BlockScaledGemmDescription const &operation_desc =
static_cast<library::BlockScaledGemmDescription const &>(operation->description());
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
PerformanceResult const &result,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size) -> std::optional<PerformanceResult> {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(gemm_workspace_, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
const auto can_implement = operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
if (can_implement != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
library::Operation const* underlying_operation = operation;
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.host_workspace.resize(workspace_size, 0);
workspace_size = underlying_operation->get_device_workspace_size(&gemm_workspace_.configuration,
&gemm_workspace_.arguments);
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
Status status = Status::kSuccess;
status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
if (status != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
curr_result.status = profile_cutlass_(
curr_result,
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
return curr_result;
};
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
for (size_t i = 0; i < 3; ++i) {
if (preferred_cluster[i] % fallback_cluster[i] != 0) {
return false;
}
}
return true;
};
// Helper function to select the best performance number among a list.
auto select_best_candidate = [&](std::vector<PerformanceResult> &candidates) {
assert(!candidates.empty() && "Candidates vector should not be empty");
auto best_iter = std::max_element(
candidates.begin(), candidates.end(),
[](PerformanceResult const &a, PerformanceResult const &b) {
return a.gflops_per_sec() < b.gflops_per_sec();
}
);
assert(best_iter != candidates.end() && "No candidate found despite non-empty candidates vector");
results_.push_back(std::move(*best_iter));
};
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
}
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
// With the fixed shape option turned on, only a specific problem shape is tested.
if (options.profiling.enable_best_kernel_for_fixed_shape) {
this->problem_.problem_sizes = {{int(this->problem_.m), int(this->problem_.n), int(this->problem_.k)}};
this->problem_.leading_dims = {{this->problem_.lda, this->problem_.ldb, this->problem_.ldc}};
}
for (int i = 0; i < int(this->problem_.problem_sizes.size()); i++) {
gemm::GemmCoord problem_shape = problem_.problem_sizes[i];
std::array<int64_t, 3> leading_dim = problem_.leading_dims[i];
auto result_opt = initialize_and_profile(result_base, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
if (result_opt) { // Only add valid results
candidates.push_back(*result_opt);
}
}
}// for raster_order
}// for swizzle_size
}// for fallback_cluster
}// for swizzle_size
if (candidates.empty()) {
return false;
}
select_best_candidate(candidates);
}
else {
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
return true;
}
tools/profiler/src/gemm_operation_profiler.cu
+363 -66
View File
@@ -53,7 +53,6 @@
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
@@ -175,7 +174,6 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
@@ -205,7 +203,6 @@ Status GemmOperationProfiler::GemmProblem::parse(
// default value
this->cluster_k_fallback = 0;
}
if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {
// default value
@@ -315,33 +312,80 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
// instantiation
int num_sizes = 8;
this->problem_sizes.resize(num_sizes);
this->leading_dims.resize(num_sizes, {0, 0, 0});
int m0 = 1024;
int n0 = 1024;
int k0 = 1024;
for (int i = 0; i < num_sizes; i++) {
auto m = m0 * (i + 1);
auto n = n0 * (i + 1);
auto k = k0 * (i + 1);
this->problem_sizes[i] = {m, n, k};
this->leading_dims[i] = {
DeviceAllocation::get_packed_layout(operation_desc.A.layout, {int(m), int(k)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.B.layout, {int(k), int(n)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.C.layout, {int(m), int(n)}).front()
};
}
this->raster_orders = {
cutlass::library::RasterOrder::kAlongN,
cutlass::library::RasterOrder::kAlongM
};
this->swizzle_sizes = {1, 2, 4, 8};
this->preferred_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}, {4, 1, 1}, {4, 2, 1}, {4, 4, 1}, {8, 2, 1}
};
this->fallback_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}
};
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
int64_t GemmOperationProfiler::GemmProblem::bytes_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_shape.m() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_shape.n() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
}
bytes *= batch_count;
return bytes;
}
/// Total number of bytes loaded
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return bytes_with_problem_shape(operation_desc, problem_shape);
}
/// Total number of flops computed
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
int64_t GemmOperationProfiler::GemmProblem::flops_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t flops_ = (int64_t(problem_shape.m()) * problem_shape.n() * problem_shape.k() + problem_shape.m() * problem_shape.n()) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
@@ -363,6 +407,12 @@ int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const
return flops_;
}
/// Total number of flops computed
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return flops_with_problem_shape(operation_desc, problem_shape);
}
/// Initializes a performance result
void GemmOperationProfiler::GemmProblem::initialize_result(
@@ -440,8 +490,9 @@ Status GemmOperationProfiler::initialize_configuration(
Status status = problem_.parse(operation_desc, problem_space, problem);
// Note: this is a temporary workaround
bool is_current_operation_sm90_mixed_dtype_shuffle = (strstr(operation_desc.name, "_shfl") != NULL);
if (is_current_operation_sm90_mixed_dtype_shuffle && (problem_.enable_sm90_mixed_dtype_shuffle_test == false)) {
bool is_sm90_operation = (strstr(operation_desc.name, "_sm90") != NULL);
bool is_sm90_mixed_dtype_shuffle_operation = (strstr(operation_desc.name, "_shfl") != NULL);
if (is_sm90_mixed_dtype_shuffle_operation && (problem_.enable_sm90_mixed_dtype_shuffle_test == false)) {
return Status::kErrorInvalidProblem;
}
@@ -457,7 +508,7 @@ Status GemmOperationProfiler::initialize_configuration(
library::NumericTypeID b_elem = library::get_real_type(operation_desc.B.element);
int a_elem_bits = library::sizeof_bits(a_elem);
int b_elem_bits = library::sizeof_bits(b_elem);
bool is_mixed_input = (a_elem_bits != b_elem_bits);
bool is_sm90_mixed_dtype_operation = is_sm90_operation && (a_elem_bits != b_elem_bits);
for (size_t i = 0; i < device_count; ++i) {
cudaSetDevice(options.device.device_id(i));
@@ -519,7 +570,7 @@ Status GemmOperationProfiler::initialize_configuration(
initialize_result_(this->model_result_, options, operation_desc, problem_space);
if (is_mixed_input)
if (is_sm90_mixed_dtype_operation)
{
const int options_g = problem_.k;
const int options_l = problem_.batch_count;
@@ -577,16 +628,14 @@ Status GemmOperationProfiler::initialize_configuration(
// Here is the first touch of the arguments, mark the mixed dtype,
// populate the scale and zero tensors in the following can_implement() call later.
// A and B are not populated at this moment, so do not update the dequantized A or B
gemm_workspace_[i].arguments.is_mixed_dtype = true;
gemm_workspace_[i].arguments.is_sm90_mixed_dtype = true;
gemm_workspace_[i].arguments.wider_operand = (a_elem_bits > b_elem_bits) ? cutlass::library::Sm90MixedInputWiderOperand::A : cutlass::library::Sm90MixedInputWiderOperand::B;
gemm_workspace_[i].arguments.generate_scale_and_zero = true;
gemm_workspace_[i].arguments.generate_dequantized_AB = false;
gemm_workspace_[i].arguments.dequantized_AB_ready = (bool *) malloc(sizeof(bool));
gemm_workspace_[i].arguments.dequantized_AB_ready[0] = false;
gemm_workspace_[i].arguments.Scale = gemm_workspace_[i].Scale->data();
gemm_workspace_[i].arguments.Zero = gemm_workspace_[i].Zero->data();
gemm_workspace_[i].arguments.packed_Scale = gemm_workspace_[i].packed_Scale->data();
} // End of "if (is_mixed_input)"
} // End of "if (is_sm90_mixed_dtype_operation)"
const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments);
if (can_implement != Status::kSuccess) {
@@ -604,6 +653,72 @@ Status GemmOperationProfiler::initialize_configuration(
return status;
}
void GemmOperationProfiler::update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
gemm_workspace.arguments.problem_size.n() = problem_shape.n();
gemm_workspace.arguments.problem_size.k() = problem_shape.k();
gemm_workspace.arguments.lda = leading_dim[0];
gemm_workspace.arguments.ldb = leading_dim[1];
gemm_workspace.arguments.ldc = leading_dim[2];
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
gemm_workspace.configuration.ldc = leading_dim[2];
}
void GemmOperationProfiler::update_result_(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
set_argument(result, "m", problem_space, problem_shape.m());
set_argument(result, "n", problem_space, problem_shape.n());
set_argument(result, "k", problem_space, problem_shape.k());
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
/// Initializes the performance result
void GemmOperationProfiler::initialize_result_(
PerformanceResult &result,
@@ -723,14 +838,25 @@ Status GemmOperationProfiler::initialize_workspace(
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
bool enable_deep_profiling = options.profiling.enable_kernel_performance_search;
int seed_shift = 0;
// When exhaustive performance search (deep profiling) option is enabled, device buffers are initialized to the largest problem shape
// so that later performance search can re-use those buffers.
int init_m = enable_deep_profiling ? std::max(int(problem_.m), problem_.problem_sizes.back().m()) : int(problem_.m);
int init_n = enable_deep_profiling ? std::max(int(problem_.n), problem_.problem_sizes.back().n()) : int(problem_.n);
int init_k = enable_deep_profiling ? std::max(int(problem_.k), problem_.problem_sizes.back().k()) : int(problem_.k);
int init_lda = enable_deep_profiling ? int(std::max(problem_.lda, problem_.leading_dims.back()[0])) : int(problem_.lda);
int init_ldb = enable_deep_profiling ? int(std::max(problem_.ldb, problem_.leading_dims.back()[1])) : int(problem_.ldb);
int init_ldc = enable_deep_profiling ? int(std::max(problem_.ldc, problem_.leading_dims.back()[2])) : int(problem_.ldc);
gemm_workspace_[i].A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
{init_m, init_k},
{init_lda},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -741,8 +867,8 @@ Status GemmOperationProfiler::initialize_workspace(
"B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
{init_k, init_n},
{init_ldb},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -753,8 +879,8 @@ Status GemmOperationProfiler::initialize_workspace(
"C",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -765,8 +891,8 @@ Status GemmOperationProfiler::initialize_workspace(
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
@@ -776,13 +902,13 @@ Status GemmOperationProfiler::initialize_workspace(
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
// Encoded tensor has the same shape and data type of the narrow data type tensor
@@ -830,14 +956,14 @@ Status GemmOperationProfiler::initialize_workspace(
i // device_index
);
}
}
} // End of "if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype)"
}
if (options.execution_mode != ExecutionMode::kDryRun) {
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_[i].arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_[i].arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
gemm_workspace_[i].arguments.lda = problem_.lda;
@@ -974,7 +1100,7 @@ bool GemmOperationProfiler::verify_cutlass(
gemm_workspace_[i].arguments.batch_stride_C = gemm_workspace_[i].C->batch_stride();
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Scale and zero already generated in initialize_configuration(),
// A and B already generated in initialize_workspace(), signal
// GemmUniversal3xOperation::update_arguments_() (trigger by underlying_operation->run())
@@ -1299,7 +1425,7 @@ bool GemmOperationProfiler::verify_with_reference_(
cutlass::library::NumericTypeID element_A_for_reference = element_A;
cutlass::library::NumericTypeID element_B_for_reference = element_B;
if (gemm_workspace_[i].arguments.is_mixed_dtype && gemm_workspace_[i].arguments.dequantized_AB_ready[0]) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
if (gemm_workspace_[i].arguments.wider_operand == cutlass::library::Sm90MixedInputWiderOperand::A) {
@@ -1444,42 +1570,213 @@ bool GemmOperationProfiler::profile(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
// Initialize structure containing GEMM arguments
gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->data();
gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->data();
gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->data();
gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->data();
gemm_workspace_[i].arguments.alpha = problem_.alpha.data();
gemm_workspace_[i].arguments.beta = problem_.beta.data();
gemm_workspace_[i].arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_[i].arguments.batch_stride_A = gemm_workspace_[i].A->batch_stride();
gemm_workspace_[i].arguments.batch_stride_B = gemm_workspace_[i].B->batch_stride();
gemm_workspace_[i].arguments.batch_stride_C = gemm_workspace_[i].C->batch_stride();
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
// Case when we either screen the best performance number of kernels with or without a fixed problem shape fed in.
if (options.profiling.enable_kernel_performance_search || options.profiling.enable_best_kernel_for_fixed_shape) {
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[i].arguments.D = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].arguments.alpha = problem_.alpha_one.data();
gemm_workspace_[i].arguments.beta = problem_.beta_zero.data();
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
gemm_workspace_[i].reduction_arguments.workspace = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.source = gemm_workspace_[i].C->data();
gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->data();
gemm_workspace_[i].reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_[i].reduction_arguments.beta = problem_.beta.data();
gemm_workspace_[i].reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
PerformanceResult const &result,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size) -> std::optional<PerformanceResult> {
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
// Initialize structure containing GEMM arguments
auto& workspace = gemm_workspace_[i];
workspace.arguments.A = workspace.A->data();
workspace.arguments.B = workspace.B->data();
workspace.arguments.C = workspace.C->data();
workspace.arguments.D = workspace.Computed->data();
workspace.arguments.alpha = problem_.alpha.data();
workspace.arguments.beta = problem_.beta.data();
workspace.arguments.pointer_mode = library::ScalarPointerMode::kHost;
workspace.arguments.batch_stride_A = workspace.A->batch_stride();
workspace.arguments.batch_stride_B = workspace.B->batch_stride();
workspace.arguments.batch_stride_C = workspace.C->batch_stride();
workspace.arguments.batch_stride_D = workspace.Computed->batch_stride();
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
workspace.arguments.D = workspace.device_workspace.data();
workspace.arguments.alpha = problem_.alpha_one.data();
workspace.arguments.beta = problem_.beta_zero.data();
workspace.reduction_arguments.workspace = workspace.device_workspace.data();
workspace.reduction_arguments.source = workspace.C->data();
workspace.reduction_arguments.destination = workspace.Computed->data();
workspace.reduction_arguments.alpha = problem_.alpha.data();
workspace.reduction_arguments.beta = problem_.beta.data();
workspace.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(workspace, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
const auto can_implement = operation->can_implement(&workspace.configuration, &workspace.arguments);
if (can_implement != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
library::Operation const* underlying_operation = operation;
cudaSetDevice(options.device.device_id(i));
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&workspace.configuration);
workspace.host_workspace.resize(workspace_size, 0);
workspace_size = underlying_operation->get_device_workspace_size(&workspace.configuration,
&workspace.arguments);
bool is_sparse = operation_desc.tile_description.math_instruction.opcode_class == cutlass::library::OpcodeClassID::kSparseTensorOp;
if (is_sparse) {
// sparse gemm get_device_workspace_size() only return device workspace size per iteration
// Needs to multiply it w/ number of iteration
workspace_size *= workspace.problem_count;
}
workspace.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
Status status = Status::kSuccess;
if (is_sparse) {
uint8_t* profiler_workspaces[1];
profiler_workspaces[0] = reinterpret_cast<uint8_t*>(workspace.A->data());
// Sparse operations have a different initialize interface.
// initialize_with_profiler_workspace converts mxk tensorA to compressed mxk/sp tensorA and the tensorE
auto modifiable_underlying_op = const_cast<library::Operation*>(underlying_operation);
status = modifiable_underlying_op->initialize_with_profiler_workspace(
&workspace.configuration,
workspace.host_workspace.data(),
workspace.device_workspace.data(),
profiler_workspaces,
workspace.problem_count,
workspace.stream);
}
else {
status = underlying_operation->initialize(
&workspace.configuration,
workspace.host_workspace.data(),
workspace.device_workspace.data(),
workspace.stream);
}
if (status != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
curr_result.status = profile_cutlass_(
curr_result,
options,
operation,
nullptr,
nullptr,
nullptr
);
return curr_result;
};
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
for (size_t i = 0; i < 3; ++i) {
if (preferred_cluster[i] % fallback_cluster[i] != 0) {
return false;
}
}
return true;
};
// Helper function to select the best performance number among a list.
auto select_best_candidate = [&](std::vector<PerformanceResult> &candidates) {
assert(!candidates.empty() && "Candidates vector should not be empty");
auto best_iter = std::max_element(
candidates.begin(), candidates.end(),
[](PerformanceResult const &a, PerformanceResult const &b) {
return a.gflops_per_sec() < b.gflops_per_sec();
}
);
assert(best_iter != candidates.end() && "No candidate found despite non-empty candidates vector");
results_.push_back(std::move(*best_iter));
};
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
}
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
// With the fixed shape option turned on, only a specific problem shape is tested.
if (options.profiling.enable_best_kernel_for_fixed_shape) {
this->problem_.problem_sizes = {{int(this->problem_.m), int(this->problem_.n), int(this->problem_.k)}};
this->problem_.leading_dims = {{this->problem_.lda, this->problem_.ldb, this->problem_.ldc}};
}
for (int i = 0; i < int(this->problem_.problem_sizes.size()); i++) {
gemm::GemmCoord problem_shape = problem_.problem_sizes[i];
std::array<int64_t, 3> leading_dim = problem_.leading_dims[i];
auto result_opt = initialize_and_profile(result_base, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
if (result_opt) { // Only add valid results
candidates.push_back(*result_opt);
}
}
}// for raster_order
}// for swizzle_size
}// for fallback_cluster
}// for swizzle_size
if (candidates.empty()) {
return false;
}
select_best_candidate(candidates);
}
// Basic case where we benchmark input parameters only.
else {
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
nullptr,
nullptr,
nullptr
);
}
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
nullptr,
nullptr,
nullptr
);
}
return true;
}
@@ -1512,7 +1809,7 @@ Status GemmOperationProfiler::profile_cutlass_(
gemm_workspace_[dev_id].arguments.C = gemm_workspace_[dev_id].C->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
if (gemm_workspace_[dev_id].arguments.is_mixed_dtype) {
if (gemm_workspace_[dev_id].arguments.is_sm90_mixed_dtype) {
// Scale, zero, and dequantized tensors are already generated in
// verify_cutlass(), no need to re-generate them in profiling
gemm_workspace_[dev_id].arguments.generate_scale_and_zero = false;
tools/profiler/src/options.cu
+5 -1
View File
@@ -478,6 +478,8 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("profiling-duration", duration, 10);
cmdline.get_cmd_line_argument("min-iterations", min_iterations, 10);
cmdline.get_cmd_line_argument("use-cuda-graphs", use_cuda_graphs, false);
cmdline.get_cmd_line_argument("enable-kernel-performance-search", enable_kernel_performance_search, false);
cmdline.get_cmd_line_argument("enable-best-kernel-for-fixed-shape", enable_best_kernel_for_fixed_shape, false);
if (cmdline.check_cmd_line_flag("providers")) {
@@ -683,7 +685,9 @@ Options::Report::Report(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("verbose", verbose, true);
cmdline.get_cmd_line_argument("sort-results", sort_results, false);
cmdline.get_cmd_line_argument("sort-results-flops-per-byte", sort_flops_per_byte, false);
cmdline.get_cmd_line_argument("sort-results-flops-per-sec", sort_flops_per_sec, false);
cmdline.get_cmd_line_argument("print-kernel-before-running", print_kernel_before_running, false);
}
tools/profiler/src/performance_report.cpp
+20 -3
View File
@@ -160,7 +160,7 @@ void PerformanceReport::append_result(PerformanceResult result) {
}
}
void PerformanceReport::sort_results(PerformanceResultVector &results) {
void PerformanceReport::sort_flops_per_byte(PerformanceResultVector &results) {
struct FlopsPerByteCompare
{
@@ -176,6 +176,19 @@ void PerformanceReport::sort_results(PerformanceResultVector &results) {
std::stable_sort(results.begin(), results.end(), FlopsPerByteCompare());
}
void PerformanceReport::sort_flops_per_sec(PerformanceResultVector &results) {
struct FlopsPerSecondCompare
{
bool operator()(const PerformanceResult &a, const PerformanceResult &b)
{
return a.gflops_per_sec() > b.gflops_per_sec();
}
};
std::stable_sort(results.begin(), results.end(), FlopsPerSecondCompare());
}
void PerformanceReport::append_results(PerformanceResultVector const &results) {
if (options_.report.verbose) {
@@ -195,8 +208,12 @@ PerformanceReport::~PerformanceReport() {
//
if (options_.report.verbose && !concatenated_results_.empty()) {
if (options_.report.sort_results) {
sort_results(concatenated_results_);
if (options_.report.sort_flops_per_byte) {
sort_flops_per_byte(concatenated_results_);
}
if (options_.report.sort_flops_per_sec) {
sort_flops_per_sec(concatenated_results_);
}
std::cout << "\n\n";