leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

v4.2 release. (#2587)

Browse Source 
* Fix default cluster callback values to 1 to avoid profiler failure when these values are not set in command line.

* v4.2 release.
This commit is contained in:
Junkai-Wu
2025-08-23 06:11:24 +08:00
committed by GitHub
parent 11cad1f67b
commit a49a78ffef
351 changed files with 28182 additions and 2032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
View File

@@ -176,7 +176,7 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
this->cluster_m = std::string(operation_desc.name).find("_2sm") != std::string::npos ? 2 : 1;
}
if (!arg_as_int(this->cluster_n, "cluster_n", problem_space, problem)) {
@@ -191,17 +191,17 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m_fallback, "cluster_m_fallback", problem_space, problem)) {
// default value
this->cluster_m_fallback = 0;
this->cluster_m_fallback = (this->cluster_m % 2 == 0) ? 2 : 1;
}
if (!arg_as_int(this->cluster_n_fallback, "cluster_n_fallback", problem_space, problem)) {
// default value
this->cluster_n_fallback = 0;
this->cluster_n_fallback = 1;
}
if (!arg_as_int(this->cluster_k_fallback, "cluster_k_fallback", problem_space, problem)) {
// default value
this->cluster_k_fallback = 0;
this->cluster_k_fallback = 1;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
@@ -540,6 +540,8 @@ Status BlockScaledGemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.use_pdl = problem_.use_pdl;
cudaStreamCreateWithFlags(&gemm_workspace_.stream, cudaStreamNonBlocking);
// initialize reduction operation for parallel splitKMode
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
if (!initialize_reduction_configuration_(operation, problem)) {
@@ -643,6 +645,7 @@ void BlockScaledGemmOperationProfiler::initialize_result_(
result.bytes = problem_.bytes(operation_desc);
result.flops = problem_.flops(operation_desc);
result.runtime = 0;
result.runtime_vector.resize(options.device.devices.size(), 0);
}
@@ -1578,7 +1581,7 @@ Status BlockScaledGemmOperationProfiler::profile_cutlass_(
}
}
auto func = [&](cudaStream_t, int iteration) {
auto func = [&](cudaStream_t stream, int iteration) {
// Iterate over copies of the problem in memory
int problem_idx = (iteration % gemm_workspace_.problem_count) * problem_.batch_count;
@@ -1599,7 +1602,7 @@ Status BlockScaledGemmOperationProfiler::profile_cutlass_(
arguments,
host_workspace,
device_workspace,
nullptr);
stream);
if (status != Status::kSuccess) {
return status;
@@ -1611,7 +1614,7 @@ Status BlockScaledGemmOperationProfiler::profile_cutlass_(
&gemm_workspace_.reduction_arguments,
gemm_workspace_.reduction_host_workspace.data(),
nullptr,
nullptr);
stream);
if (status != Status::kSuccess) {
return status;
@@ -1621,7 +1624,7 @@ Status BlockScaledGemmOperationProfiler::profile_cutlass_(
return status;
};
return profile_kernel_(result, options, func);
return profile_kernel_(result, options, func, gemm_workspace_.stream);
}
/////////////////////////////////////////////////////////////////////////////////////////////////

362
tools/profiler/src/blockwise_gemm_operation_profiler.cu
View File

@@ -116,21 +116,21 @@ void BlockwiseGemmOperationProfiler::print_examples(std::ostream &out) const {
<< " $ cutlass_profiler --operation=blockwise_gemm --m=1024:4096:256 --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
<< "For column major, use column, col, or n. For row major use, row or t:\n"
<< " $ cutlass_profiler --operation=Gemm --A=f16:column --B=*:row\n\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --A=f16:column --B=*:row\n\n"
<< "Profile a particular problem size with split K and parallel reduction:\n"
<< " $ cutlass_profiler --operation=Gemm --split_k_mode=parallel --split_k_slices=2 --m=1024 --n=1024 --k=128\n\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --split_k_mode=parallel --split_k_slices=2 --m=1024 --n=1024 --k=128\n\n"
<< "Using various input value distribution:\n"
<< " $ cutlass_profiler --operation=Gemm --dist=uniform,min:0,max:3\n"
<< " $ cutlass_profiler --operation=Gemm --dist=gaussian,mean:0,stddev:3\n"
<< " $ cutlass_profiler --operation=Gemm --dist=sequential,start:0,delta:1\n\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --dist=uniform,min:0,max:3\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --dist=gaussian,mean:0,stddev:3\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --dist=sequential,start:0,delta:1\n\n"
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=Gemm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< " $ cutlass_profiler --operation=blockwise_gemm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to gemm kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=Gemm \\ \n"
<< " $ cutlass_profiler --operation=blockwise_gemm \\ \n"
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
@@ -194,7 +194,7 @@ Status BlockwiseGemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
this->cluster_m = std::string(operation_desc.name).find("_2sm") != std::string::npos ? 2 : 1;
}
if (!arg_as_int(this->cluster_n, "cluster_n", problem_space, problem)) {
@@ -209,17 +209,17 @@ Status BlockwiseGemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m_fallback, "cluster_m_fallback", problem_space, problem)) {
// default value
this->cluster_m_fallback = 0;
this->cluster_m_fallback = (this->cluster_m % 2 == 0) ? 2 : 1;
}
if (!arg_as_int(this->cluster_n_fallback, "cluster_n_fallback", problem_space, problem)) {
// default value
this->cluster_n_fallback = 0;
this->cluster_n_fallback = 1;
}
if (!arg_as_int(this->cluster_k_fallback, "cluster_k_fallback", problem_space, problem)) {
// default value
this->cluster_k_fallback = 0;
this->cluster_k_fallback = 1;
}
@@ -331,33 +331,80 @@ Status BlockwiseGemmOperationProfiler::GemmProblem::parse(
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
// instantiation
int num_sizes = 8;
this->problem_sizes.resize(num_sizes);
this->leading_dims.resize(num_sizes, {0, 0, 0});
int m0 = 1024;
int n0 = 1024;
int k0 = 1024;
for (int i = 0; i < num_sizes; i++) {
auto m = m0 * (i + 1);
auto n = n0 * (i + 1);
auto k = k0 * (i + 1);
this->problem_sizes[i] = {m, n, k};
this->leading_dims[i] = {
DeviceAllocation::get_packed_layout(operation_desc.A.layout, {int(m), int(k)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.B.layout, {int(k), int(n)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.C.layout, {int(m), int(n)}).front()
};
}
this->swizzle_sizes = {1, 2, 4, 8};
this->preferred_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}, {4, 1, 1}, {4, 2, 1}, {4, 4, 1}, {8, 2, 1}
};
this->fallback_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}
};
this->raster_orders = {
cutlass::library::RasterOrder::kAlongN,
cutlass::library::RasterOrder::kAlongM
};
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t BlockwiseGemmOperationProfiler::GemmProblem::bytes(library::BlockwiseGemmDescription const &operation_desc) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t BlockwiseGemmOperationProfiler::GemmProblem::bytes_with_problem_shape(
library::BlockwiseGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_shape.m() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_shape.n() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n() +
int64_t(library::sizeof_bits(operation_desc.SFA.element) * problem_shape.m() / operation_desc.SFMVecSize / 8) * problem_shape.k() / operation_desc.SFKVecSize +
int64_t(library::sizeof_bits(operation_desc.SFB.element) * problem_shape.n() / operation_desc.SFNVecSize / 8) * problem_shape.k() / operation_desc.SFKVecSize;
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
}
bytes *= batch_count;
return bytes;
}
/// Total number of bytes loaded
int64_t BlockwiseGemmOperationProfiler::GemmProblem::bytes(library::BlockwiseGemmDescription const &operation_desc) const {
return bytes_with_problem_shape(operation_desc, {int(m), int(n), int(k)});
}
/// Total number of flops computed
int64_t BlockwiseGemmOperationProfiler::GemmProblem::flops(library::BlockwiseGemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
int64_t BlockwiseGemmOperationProfiler::GemmProblem::flops_with_problem_shape(
library::BlockwiseGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t flops_ = (int64_t(problem_shape.m()) * problem_shape.n() * problem_shape.k() + problem_shape.m() * problem_shape.n()) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
@@ -379,6 +426,10 @@ int64_t BlockwiseGemmOperationProfiler::GemmProblem::flops(library::BlockwiseGem
return flops_;
}
/// Total number of flops computed
int64_t BlockwiseGemmOperationProfiler::GemmProblem::flops(library::BlockwiseGemmDescription const &operation_desc) const {
return flops_with_problem_shape(operation_desc, {int(m), int(n), int(k)});
}
/// Initializes a performance result
void BlockwiseGemmOperationProfiler::GemmProblem::initialize_result(
@@ -1185,42 +1236,249 @@ bool BlockwiseGemmOperationProfiler::profile(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
// Case when we either screen the best performance number of kernels with or without a fixed problem shape fed in.
if (options.profiling.enable_kernel_performance_search || options.profiling.enable_best_kernel_for_fixed_shape) {
library::BlockwiseGemmDescription const &operation_desc =
static_cast<library::BlockwiseGemmDescription const &>(operation->description());
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
auto cluster_shape = operation_desc.tile_description.cluster_shape;
bool is_dynamic_cluster_enabled = cluster_shape.m() == 0 || cluster_shape.n() == 0 || cluster_shape.k() == 0;
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
PerformanceResult const &result,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size) -> std::optional<PerformanceResult> {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
gemm_workspace_.arguments.problem_size.m() = problem_shape.m();
gemm_workspace_.arguments.problem_size.n() = problem_shape.n();
gemm_workspace_.arguments.problem_size.k() = problem_shape.k();
gemm_workspace_.arguments.lda = leading_dim[0];
gemm_workspace_.arguments.ldb = leading_dim[1];
gemm_workspace_.arguments.ldc = leading_dim[2];
gemm_workspace_.arguments.swizzle_size = swizzle_size;
gemm_workspace_.arguments.raster_order = raster_order;
if (is_dynamic_cluster_enabled) {
gemm_workspace_.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace_.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace_.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace_.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
}
gemm_workspace_.configuration.problem_size.m() = problem_shape.m();
gemm_workspace_.configuration.problem_size.n() = problem_shape.n();
gemm_workspace_.configuration.problem_size.k() = problem_shape.k();
gemm_workspace_.configuration.lda = leading_dim[0];
gemm_workspace_.configuration.ldb = leading_dim[1];
gemm_workspace_.configuration.ldc = leading_dim[2];
const auto can_implement = operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
if (can_implement != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
library::Operation const* underlying_operation = operation;
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.host_workspace.resize(workspace_size, 0);
workspace_size = underlying_operation->get_device_workspace_size(&gemm_workspace_.configuration,
&gemm_workspace_.arguments);
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
Status status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data(),
nullptr);
if (status != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
PerformanceResult curr_result(result);
curr_result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
curr_result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
set_argument(curr_result, "m", problem_space, problem_shape.m());
set_argument(curr_result, "n", problem_space, problem_shape.n());
set_argument(curr_result, "k", problem_space, problem_shape.k());
set_argument(curr_result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(curr_result, "swizzle_size", problem_space, swizzle_size);
if (is_dynamic_cluster_enabled) {
set_argument(curr_result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(curr_result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(curr_result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(curr_result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(curr_result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(curr_result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
curr_result.status = profile_cutlass_(
curr_result,
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
return curr_result;
};
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
for (size_t i = 0; i < 3; ++i) {
if (preferred_cluster[i] % fallback_cluster[i] != 0) {
return false;
}
}
return true;
};
// Helper function to select the best performance number among a list.
auto select_best_candidate = [&](std::vector<PerformanceResult> &candidates) {
assert(!candidates.empty() && "Candidates vector should not be empty");
auto best_iter = std::max_element(
candidates.begin(), candidates.end(),
[](PerformanceResult const &a, PerformanceResult const &b) {
return a.gflops_per_sec() < b.gflops_per_sec();
}
);
assert(best_iter != candidates.end() && "No candidate found despite non-empty candidates vector");
results_.push_back(std::move(*best_iter));
};
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
}
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (is_dynamic_cluster_enabled && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
// With the fixed shape option turned on, only a specific problem shape is tested.
if (options.profiling.enable_best_kernel_for_fixed_shape) {
this->problem_.problem_sizes = {{int(this->problem_.m), int(this->problem_.n), int(this->problem_.k)}};
this->problem_.leading_dims = {{this->problem_.lda, this->problem_.ldb, this->problem_.ldc}};
}
for (int i = 0; i < int(this->problem_.problem_sizes.size()); i++) {
gemm::GemmCoord problem_shape = problem_.problem_sizes[i];
std::array<int64_t, 3> leading_dim = problem_.leading_dims[i];
auto result_opt = initialize_and_profile(result_base, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
if (result_opt) { // Only add valid results
candidates.push_back(*result_opt);
}
}
}// for raster_order
}// for swizzle_size
}// for fallback_cluster
}// for swizzle_size
if (candidates.empty()) {
return false;
}
select_best_candidate(candidates);
}
else {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
}
return true;
}

8
tools/profiler/src/gemm_operation_profiler.cu
View File

@@ -176,7 +176,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
this->cluster_m = std::string(operation_desc.name).find("_2sm") != std::string::npos ? 2 : 1;
}
if (!arg_as_int(this->cluster_n, "cluster_n", problem_space, problem)) {
@@ -191,17 +191,17 @@ Status GemmOperationProfiler::GemmProblem::parse(
if (!arg_as_int(this->cluster_m_fallback, "cluster_m_fallback", problem_space, problem)) {
// default value
this->cluster_m_fallback = 0;
this->cluster_m_fallback = (this->cluster_m % 2 == 0) ? 2 : 1;
}
if (!arg_as_int(this->cluster_n_fallback, "cluster_n_fallback", problem_space, problem)) {
// default value
this->cluster_n_fallback = 0;
this->cluster_n_fallback = 1;
}
if (!arg_as_int(this->cluster_k_fallback, "cluster_k_fallback", problem_space, problem)) {
// default value
this->cluster_k_fallback = 0;
this->cluster_k_fallback = 1;
}
if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {

20
tools/profiler/src/grouped_gemm_operation_profiler.cu
View File

@@ -283,7 +283,7 @@ Status GroupedGemmOperationProfiler::GroupedGemmProblem::parse(
if (!arg_as_int(this->cluster_m_fallback, "cluster_m_fallback", problem_space, problem)) {
// default value
this->cluster_m_fallback = std::string(operation_desc.gemm.name).find("_2sm") != std::string::npos ? 2 : 1;
this->cluster_m_fallback = (this->cluster_m % 2 == 0) ? 2 : 1;
}
if (!arg_as_int(this->cluster_n_fallback, "cluster_n_fallback", problem_space, problem)) {
@@ -408,6 +408,11 @@ int64_t GroupedGemmOperationProfiler::GroupedGemmProblem::bytes(
for (size_t group_idx = 0, num_groups = problem_sizes.size(); group_idx < num_groups;
group_idx++) {
// If M = 0 or N = 0, no tiles are scheduled and no bytes are loaded for the group
if (m(group_idx) * n(group_idx) == 0) {
continue;
}
bytes +=
int64_t(library::sizeof_bits(operation_desc.gemm.A.element) * m(group_idx) / 8) * k(group_idx) +
int64_t(library::sizeof_bits(operation_desc.gemm.B.element) * n(group_idx) / 8) * k(group_idx) +
@@ -630,6 +635,8 @@ Status GroupedGemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.use_pdl = problem_.use_pdl;
cudaStreamCreateWithFlags(&gemm_workspace_.stream, cudaStreamNonBlocking);
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return status;
@@ -654,6 +661,7 @@ void GroupedGemmOperationProfiler::initialize_result_(
result.bytes = problem_.bytes(operation_desc);
result.flops = problem_.flops(operation_desc);
result.runtime = 0;
result.runtime_vector.resize(options.device.devices.size(), 0);
}
@@ -1585,9 +1593,9 @@ Status GroupedGemmOperationProfiler::profile_cutlass_(
void* host_workspace,
void* device_workspace) {
library::Operation const* underlying_operation = operation;
results_.back().status = underlying_operation->initialize_with_arguments(&gemm_workspace_.arguments);
if (results_.back().status != Status::kSuccess) {
return results_.back().status;
result.status = underlying_operation->initialize_with_arguments(&gemm_workspace_.arguments);
if (result.status != Status::kSuccess) {
return result.status;
}
auto func = [&](cudaStream_t stream, int iteration) {
@@ -1600,9 +1608,9 @@ Status GroupedGemmOperationProfiler::profile_cutlass_(
gemm_workspace_.arguments.ptr_C = gemm_workspace_.C_ptr_array_device[problem_idx]->data();
gemm_workspace_.arguments.ptr_D = gemm_workspace_.D_ptr_array_device[problem_idx]->data();
return underlying_operation->run(arguments, host_workspace, device_workspace);
return underlying_operation->run(arguments, host_workspace, device_workspace, stream);
};
return profile_kernel_(result, options, func);
return profile_kernel_(result, options, func, gemm_workspace_.stream);
}
/////////////////////////////////////////////////////////////////////////////////////////////////

373
tools/profiler/src/operation_profiler.cu
View File

@@ -348,132 +348,127 @@ int OperationProfiler::profile_all(
Options const &options,
library::Manifest const &manifest,
DeviceContext &device_context) {
ProblemSpace problem_space(arguments_, options.cmdline);
ProblemSpace cmdline_problem_space(arguments_, options.cmdline);
bool do_testlist_run = !options.operation_problems.empty();
std::vector<std::pair<std::string, std::unique_ptr<ProblemSpace>>> all_operations_and_problems;
if (do_testlist_run) {
for (const auto& [operation_name, cmd_vec] : options.operation_problems) {
for (auto& cmd_line : cmd_vec) {
all_operations_and_problems.push_back({operation_name, std::make_unique<ProblemSpace>(arguments_, cmd_line)});
}
}
}
// 1. Construct performance report
PerformanceReport report(options, problem_space.argument_names(), kind_);
PerformanceReport report(options, cmdline_problem_space.argument_names(), kind_);
// 2. For each problem in problem space
ProblemSpace::Iterator problem_it = problem_space.begin();
ProblemSpace::Iterator problem_end = problem_space.end();
bool continue_profiling = true;
//
int retval = 0;
// For each problem in problem space
for (; continue_profiling && problem_it != problem_end; ++problem_it) {
ProblemSpace::Problem problem = problem_it.at();
report.next_problem();
size_t bound = (all_operations_and_problems.empty() ? 1 : all_operations_and_problems.size());
for (size_t i = 0; i < bound; i++) {
// For each operation in manifest
int matched_operation_count = 0;
int profiled_operation_count = 0;
for (auto const& operation_ptr : manifest) {
// New problem space for each operation if we are running a testlist
ProblemSpace& problem_space = do_testlist_run ? *all_operations_and_problems[i].second : cmdline_problem_space;
library::Operation const *operation = operation_ptr.get();
// 2. For each problem in problem space
ProblemSpace::Iterator problem_it = problem_space.begin();
ProblemSpace::Iterator problem_end = problem_space.end();
bool continue_profiling = true;
// For each problem in problem space
for (; continue_profiling && problem_it != problem_end; ++problem_it) {
ProblemSpace::Problem problem = problem_it.at();
report.next_problem();
// For each operation in manifest
int matched_operation_count = 0;
int profiled_operation_count = 0;
for (auto const& operation_ptr : manifest) {
library::Operation const *operation = operation_ptr.get();
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " Operation: " << typeid(*operation).name() << "\n"
<< " name: " << operation->description().name << "\n"
<< " kind: " << operation->description().kind << "\n"
<< " provider: " << operation->description().provider << "\n";
std::cerr << " Operation: " << typeid(*operation).name() << "\n"
<< " name: " << operation->description().name << "\n"
<< " kind: " << operation->description().kind << "\n"
<< " provider: " << operation->description().provider << "\n";
#endif // CUTLASS_DEBUG_TRACE_LEVEL
auto min_cc = operation->description().tile_description.minimum_compute_capability;
auto max_cc = operation->description().tile_description.maximum_compute_capability;
auto min_cc = operation->description().tile_description.minimum_compute_capability;
auto max_cc = operation->description().tile_description.maximum_compute_capability;
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " min_cc: " << min_cc << "\n";
std::cerr << " max_cc: " << min_cc << "\n";
std::cerr << " min_cc: " << min_cc << "\n";
std::cerr << " max_cc: " << min_cc << "\n";
#endif
// Clear named allocations
device_context.free();
// Clear named allocations
device_context.free();
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
if (operation->description().kind != kind_) {
std::cerr << " @ kind " << operation->description().kind
<< " != kind_ " << kind_ << "\n";
}
if (operation->description().provider != library::Provider::kCUTLASS) {
std::cerr << " @ provider " << operation->description().provider
<< " != library::Provider::kCUTLASS\n";
}
if (options.device.compute_capability(0) < min_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability(0)
<< " < min_cc " << min_cc << "\n";
}
if (options.device.compute_capability(0) > max_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability(0)
<< " > max_cc " << max_cc << "\n";
}
if (operation->description().kind != kind_) {
std::cerr << " @ kind " << operation->description().kind
<< " != kind_ " << kind_ << "\n";
}
if (operation->description().provider != library::Provider::kCUTLASS) {
std::cerr << " @ provider " << operation->description().provider
<< " != library::Provider::kCUTLASS\n";
}
if (options.device.compute_capability(0) < min_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability(0)
<< " < min_cc " << min_cc << "\n";
}
if (options.device.compute_capability(0) > max_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability(0)
<< " > max_cc " << max_cc << "\n";
}
#endif
// Execute compatible cutlass operations if they satisfy the current device's compute capability
if (operation->description().kind == kind_ &&
operation->description().provider == library::Provider::kCUTLASS &&
options.device.compute_capability(0) >= min_cc &&
options.device.compute_capability(0) <= max_cc) {
// Execute compatible cutlass operations if they satisfy the current device's compute capability
if (operation->description().kind == kind_ &&
operation->description().provider == library::Provider::kCUTLASS &&
options.device.compute_capability(0) >= min_cc &&
options.device.compute_capability(0) <= max_cc) {
std::string operation_name(operation->description().name);
// Filter kernels by name
bool filtered_by_name = options.operation_names.empty();
if (!filtered_by_name) {
std::string operation_name(operation->description().name);
// Filter kernels by name
bool filtered_by_name = options.operation_names.empty();
if (!filtered_by_name) {
for (auto const & op_name : options.operation_names) {
for (auto const & op_name : options.operation_names) {
if (find_string_matches_(op_name, operation_name)) {
filtered_by_name = true;
break;
}
}
}
for (auto const & op_name : options.excluded_operation_names) {
if (find_string_matches_(op_name, operation_name)) {
filtered_by_name = true;
filtered_by_name = false;
break;
}
}
}
for (auto const & op_name : options.excluded_operation_names) {
if (find_string_matches_(op_name, operation_name)) {
// Problems list uses exact match on operation names
if (do_testlist_run && !(all_operations_and_problems[i].first == operation_name)) {
filtered_by_name = false;
break;
}
}
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
continue;
}
// we have found a kernel match, so increment the counter for match kernels
++matched_operation_count;
// A. Initialize configuration
Status status = this->initialize_configuration(
options,
report,
device_context,
operation,
problem_space,
problem);
if (status == Status::kErrorInternal) {
// If there was an internal error, consume the CUDA error and move to the next operation.
(void)cudaGetLastError();
report.append_result(model_result_);
continue;
}
else if (status != Status::kSuccess) {
// If the workspace could not be initialized for any other reason, continue to
// the next operation.
continue;
}
if (continue_profiling) {
if (options.report.print_kernel_before_running) {
std::cout << "Profiling kernel for JUnit test " << options.report.junit_output_path << ": "
<< operation_name << std::endl;
}
status = this->initialize_workspace(
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
continue;
}
// we have found a kernel match, so increment the counter for match kernels
++matched_operation_count;
// A. Initialize configuration
Status status = this->initialize_configuration(
options,
report,
device_context,
@@ -486,7 +481,7 @@ int OperationProfiler::profile_all(
// If there was an internal error, consume the CUDA error and move to the next operation.
(void)cudaGetLastError();
report.append_results(results_);
report.append_result(model_result_);
continue;
}
else if (status != Status::kSuccess) {
@@ -494,93 +489,123 @@ int OperationProfiler::profile_all(
// the next operation.
continue;
}
}
//
// Profile CUTLASS if it is enabled
//
if (continue_profiling) {
// B. Verify CUTLASS
if (continue_profiling && options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
if (options.report.print_kernel_before_running) {
std::cout << "Profiling kernel for JUnit test " << options.report.junit_output_path << ": "
<< operation_name << std::endl;
}
continue_profiling = this->verify_cutlass(
options,
report,
device_context,
operation,
problem_space,
problem);
status = this->initialize_workspace(
options,
report,
device_context,
operation,
problem_space,
problem);
retval |= (not continue_profiling);
}
if (status == Status::kErrorInternal) {
// If there was an internal error, consume the CUDA error and move to the next operation.
(void)cudaGetLastError();
report.append_results(results_);
continue;
}
else if (status != Status::kSuccess) {
// If the workspace could not be initialized for any other reason, continue to
// the next operation.
continue;
}
}
//
// Profile CUTLASS if it is enabled
//
// B. Verify CUTLASS
if (continue_profiling && options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
continue_profiling = this->verify_cutlass(
options,
report,
device_context,
operation,
problem_space,
problem);
retval |= (not continue_profiling);
}
if (options.execution_mode == ExecutionMode::kDryRun) {
report.append_results(results_);
results_.clear();
continue;
}
//
// C. Optionally save workspace
//
if (options.verification.save_workspace == SaveWorkspace::kAlways) {
save_workspace(
device_context,
options,
operation->description(),
library::Provider::kCUTLASS);
}
//
// D. Profile
//
if (continue_profiling && options.profiling.enabled) {
continue_profiling = this->profile(
options,
report,
device_context,
operation,
problem_space,
problem);
// Count op as profiled, even it failed to profile
profiled_operation_count++;
}
if (options.execution_mode == ExecutionMode::kDryRun) {
report.append_results(results_);
results_.clear();
continue;
} // if op satisfied compute capacity
if (!continue_profiling) {
// break out of `for op in manifest` loop and move to next problem
// `for each problem in problem space` conditional check on not continue profiling
break;
}
} // for op in manifest
//
// C. Optionally save workspace
//
if (options.verification.save_workspace == SaveWorkspace::kAlways) {
save_workspace(
device_context,
options,
operation->description(),
library::Provider::kCUTLASS);
}
//
// D. Profile
//
if (continue_profiling && options.profiling.enabled) {
continue_profiling = this->profile(
options,
report,
device_context,
operation,
problem_space,
problem);
// Count op as profiled, even it failed to profile
profiled_operation_count++;
}
report.append_results(results_);
results_.clear();
} // if op satisfied compute capacity
if (!continue_profiling) {
// break out of `for op in manifest` loop and move to next problem
// `for each problem in problem space` conditional check on not continue profiling
break;
// If we did not find any kernels that match our filters and error_on_no_match was set, report an error
if (options.profiling.error_on_no_match && matched_operation_count <= 0) {
#if !NDEBUG
std::cerr << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
#endif
retval |= 1;
// Stop profiling on error no match
continue_profiling = false;
}
} // for op in manifest
// If we did not find any kernels that match our filters and error_on_no_match was set, report an error
if (options.profiling.error_on_no_match && matched_operation_count <= 0) {
#if !NDEBUG
std::cerr << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
#endif
retval |= 1;
// Stop profiling on error no match
continue_profiling = false;
}
if (options.profiling.error_if_nothing_is_profiled && options.profiling.enabled && profiled_operation_count <= 0) {
#if !NDEBUG
std::cerr << "Error: No kernels profiled found with kernel selection filters [--error_if_nothing_is_profiled]" << std::endl;
#endif
retval |= 1;
// Stop profiling on error no match
continue_profiling = false;
}
if (options.profiling.error_if_nothing_is_profiled && options.profiling.enabled && profiled_operation_count <= 0) {
#if !NDEBUG
std::cerr << "Error: No kernels profiled found with kernel selection filters [--error_if_nothing_is_profiled]" << std::endl;
#endif
retval |= 1;
// Stop profiling on error no match
continue_profiling = false;
}
} // for each problem in problem space
} // for each problem in problem space
}
return retval;
}

56
tools/profiler/src/options.cu
View File

@@ -550,6 +550,9 @@ void Options::Profiling::print_usage(std::ostream &out) const {
<< " --profiling-enabled=<bool> "
<< " If true, profiling is actually conducted.\n\n"
<< " --enable-best-kernel-for-fixed-shape=<bool> "
<< " If true, iterate through common cluster sizes, raster orders, and swizzle sizes for each kernel.\n\n"
;
}
@@ -593,6 +596,9 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
if (enabled) {
cmdline.get_cmd_line_argument("verification-required", required, false);
}
else {
required = false;
}
cmdline.get_cmd_line_argument("epsilon", epsilon, 0.05);
@@ -847,6 +853,52 @@ Options::Options(cutlass::CommandLine const &cmdline):
for (std::string line; getline(input, line);) {
operation_names.push_back(line);
}
} else if (cmdline.check_cmd_line_flag("testlist-file")) {
// Problems file is a CSV, where the first column is the kernel name and the rest are the problem arguments
std::string filename;
cmdline.get_cmd_line_argument("testlist-file", filename, {});
std::ifstream input(filename);
if (!input.good()) {
throw std::runtime_error("failed to open: " + filename);
}
std::string line;
std::vector<std::string> col_names;
// Read header line
if (std::getline(input, line)) {
std::stringstream ss(line);
std::string header;
while (std::getline(ss, header, ',')) {
col_names.push_back(header);
}
}
// Read content lines
while (std::getline(input, line)) {
std::stringstream ss(line);
std::string item;
size_t colIdx = 0;
std::string operation_name;
std::unordered_map<std::string, std::string> arguments;
while (std::getline(ss, item, ',')) {
if (!colIdx) {
// First column is operation name
if (operation_problems.find(item) == operation_problems.end()) {
operation_names.push_back(item);
}
operation_name = item;
} else {
if (colIdx < col_names.size()) {
arguments[col_names[colIdx]] = item;
}
}
colIdx++;
}
operation_problems[operation_name].emplace_back(arguments);
}
}
if (cmdline.check_cmd_line_flag("ignore-kernels")) {
@@ -899,6 +951,10 @@ void Options::print_usage(std::ostream &out) const {
<< " --ignore-kernels=<string_list> "
<< " Excludes kernels whose names match anything in this list.\n\n"
<< " --testlist-file=<filename> "
<< " A CSV, where each row is a problem, where the first column is the kernel name and the rest are the problem arguments" << end_of_line
<< " The column names should match cutlass_profiler cmd line arguments. \n\n"
;
//