leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

v3.8.0 update (#2082)

Browse Source 
* 3.8 update

* fix Markus' name

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
This commit is contained in:
Yujia Zhai
2025-02-06 21:33:40 -05:00
committed by GitHub
co-authored by yuzhai
parent affd1b693d
commit 833f6990e0
168 changed files with 24945 additions and 3436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
+1 -11
View File
@@ -75,7 +75,6 @@ BlockScaledGemmOperationProfiler::BlockScaledGemmOperationProfiler(Options const
{ArgumentTypeID::kTensor, {"D"}, "Tensor storing the D output"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
// TODO: Bring these back once SM100 future audits are complete
{ArgumentTypeID::kEnumerated, {"split_k_mode", "split-k-mode"}, "Variant of split K mode(serial, parallel)"},
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
@@ -113,14 +112,11 @@ void BlockScaledGemmOperationProfiler::print_examples(std::ostream &out) const {
<< "Schmoo over problem size and beta:\n"
<< " $ cutlass_profiler --operation=block_scaled_gemm --m=1024:4096:256 --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
// TODO: Bring these back once SM100 future audits are complete
#if 0
<< "Run when A is f16 with column-major and B is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
<< "For column major, use column, col, or n. For row major use, row or t:\n"
<< " $ cutlass_profiler --operation=Gemm --A=f16:column --B=*:row\n\n"
<< "Profile a particular problem size with split K and parallel reduction:\n"
<< " $ cutlass_profiler --operation=Gemm --split_k_mode=parallel --split_k_slices=2 --m=1024 --n=1024 --k=128\n\n"
#endif
<< "Using various input value distribution:\n"
<< " $ cutlass_profiler --operation=Gemm --dist=uniform,min:0,max:3\n"
@@ -225,7 +221,6 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
this->split_k_slices = 1;
}
// TODO: Bring these back once SM100 future audits are complete
if (this->split_k_mode != library::SplitKMode::kSerial) {
std::cout<<"SplitK/StreamK feature is not supported yet!";
return Status::kErrorInvalidProblem;
@@ -403,7 +398,6 @@ void BlockScaledGemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "cluster_k_fallback", problem_space, cluster_k_fallback);
// TODO: Bring these back once SM100 future audits are complete
set_argument(result, "split_k_mode", problem_space, library::to_string(split_k_mode));
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
@@ -536,8 +530,6 @@ bool BlockScaledGemmOperationProfiler::initialize_reduction_configuration_(
library::Operation const *operation,
ProblemSpace::Problem const &problem) {
// TODO: Bring these back once SM100 future audits are complete
#if 1
library::BlockScaledGemmDescription const &gemm_desc =
static_cast<library::BlockScaledGemmDescription const&>(operation->description());
@@ -577,8 +569,6 @@ bool BlockScaledGemmOperationProfiler::initialize_reduction_configuration_(
// reduction operation found and initialized
return true;
#endif
return false;
}
/// Initializes workspace
tools/profiler/src/cublas_helpers.cu
+3
View File
@@ -545,6 +545,7 @@ bool cublasLtGemmExDispatcher::get_cublaslt_algo(cublasLtHandle_t handle,
cublasLtMatmulAlgoGetHeuristic(handle, operationDesc, Adesc, Bdesc, Cdesc, Ddesc, preference, requestedAlgoCount, heuristicResult, &returnedResults);
if (returnedResults == 0) {
cudaFree(workspaceHeuristic);
return false;
}
@@ -589,6 +590,7 @@ bool cublasLtGemmExDispatcher::get_cublaslt_algo(cublasLtHandle_t handle,
// Handle errors
if (status != CUBLAS_STATUS_SUCCESS) {
std::cerr << "cublasLtMatmul AutoTuning failed with status: " << cublasLtGetStatusName(status) << std::endl;
cudaFree(workspaceHeuristic);
return false;
}
@@ -653,6 +655,7 @@ bool cublasLtGemmExDispatcher::get_cublaslt_algo(cublasLtHandle_t handle,
throw std::bad_alloc();
}
cudaFree(workspaceHeuristic);
return true;
}
tools/profiler/src/cutlass_profiler.cu
+11 -7
View File
@@ -36,16 +36,17 @@
#include <stdexcept>
// Profiler includes
#include "cutlass/profiler/cutlass_profiler.h"
#include "cutlass/profiler/gemm_operation_profiler.h"
#include "cutlass/profiler/block_scaled_gemm_operation_profiler.h"
#include "cutlass/profiler/rank_k_operation_profiler.h"
#include "cutlass/profiler/rank_2k_operation_profiler.h"
#include "cutlass/profiler/trmm_operation_profiler.h"
#include "cutlass/profiler/symm_operation_profiler.h"
#include "cutlass/profiler/block_scaled_gemm_operation_profiler.h"
#include "cutlass/profiler/conv2d_operation_profiler.h"
#include "cutlass/profiler/conv3d_operation_profiler.h"
#include "cutlass/profiler/cutlass_profiler.h"
#include "cutlass/profiler/gemm_operation_profiler.h"
#include "cutlass/profiler/grouped_gemm_operation_profiler.h"
#include "cutlass/profiler/rank_2k_operation_profiler.h"
#include "cutlass/profiler/rank_k_operation_profiler.h"
#include "cutlass/profiler/sparse_gemm_operation_profiler.h"
#include "cutlass/profiler/symm_operation_profiler.h"
#include "cutlass/profiler/trmm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -76,6 +77,8 @@ CutlassProfiler::CutlassProfiler(
operation_profilers_.emplace_back(new TrmmOperationProfiler(options));
operation_profilers_.emplace_back(new SymmOperationProfiler(options));
operation_profilers_.emplace_back(new GroupedGemmOperationProfiler(options));
}
CutlassProfiler::~CutlassProfiler() {
@@ -201,6 +204,7 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
<< " $ cutlass_profiler --operation=SparseGemm --help\n\n"
<< " $ cutlass_profiler --operation=GroupedGemm --help\n\n"
;
}
tools/profiler/src/device_allocation.cu
-16
View File
@@ -616,7 +616,6 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::device::BlockFillRandom<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(pointer_),
@@ -657,7 +656,6 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::device::BlockFillRandom<double>(
reinterpret_cast<double *>(pointer_),
@@ -823,7 +821,6 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillRandom<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
@@ -856,7 +853,6 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillRandom<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -1086,7 +1082,6 @@ void DeviceAllocation::initialize_sequential_device(Distribution dist) {
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(pointer_),
@@ -1119,7 +1114,6 @@ void DeviceAllocation::initialize_sequential_device(Distribution dist) {
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::device::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(pointer_),
@@ -1360,7 +1354,6 @@ void DeviceAllocation::initialize_sequential_host(Distribution dist) {
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
@@ -1393,7 +1386,6 @@ void DeviceAllocation::initialize_sequential_host(Distribution dist) {
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -1690,7 +1682,6 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
@@ -1717,7 +1708,6 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float_e2m1_t const *>(ptr_A),
reinterpret_cast<float_e2m1_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF16:
return reference::device::BlockCompareEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -1886,7 +1876,6 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<float_e5m2_t>(epsilon),
static_cast<float_e5m2_t>(nonzero_floor));
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareRelativelyEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
@@ -1925,7 +1914,6 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<float_e2m1_t>(epsilon),
static_cast<float_e2m1_t>(nonzero_floor));
case library::NumericTypeID::kF16:
return reference::device::BlockCompareRelativelyEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -2273,7 +2261,6 @@ void DeviceAllocation::write_tensor_csv(
write_tensor_csv_static_type<float_ue4m3_t>(out, *this);
break;
case library::NumericTypeID::kFE2M3:
write_tensor_csv_static_type<float_e2m3_t>(out, *this);
break;
@@ -2288,7 +2275,6 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kFUE8M0:
write_tensor_csv_static_type<float_ue8m0_t>(out, *this);
break;
case library::NumericTypeID::kF16:
write_tensor_csv_static_type<half_t>(out, *this);
break;
@@ -2475,7 +2461,6 @@ void DeviceAllocation::fill_device(double val = 0.0) {
case library::NumericTypeID::kFE2M1:
tensor_fill<float_e2m1_t>(*this, static_cast<float_e2m1_t>(val));
break;
case library::NumericTypeID::kF16:
tensor_fill<half_t>(*this, static_cast<half_t>(val));
@@ -2611,7 +2596,6 @@ void DeviceAllocation::fill_host(double val = 0.0) {
static_cast<float_e2m1_t>(val)
);
break;
case library::NumericTypeID::kFE4M3:
cutlass::reference::host::BlockFill<float_e4m3_t>(
tools/profiler/src/device_context.cu
+32 -23
View File
@@ -75,6 +75,35 @@ DeviceAllocation *DeviceContext::allocate_tensor(
return allocation;
}
static void initialize_allocation_with_data_distribution(
Options const &options,
int seed_shift,
DeviceAllocation *allocation,
Distribution &data_distribution) {
if (options.initialization.provider == library::Provider::kReferenceDevice) {
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_device(
data_distribution);
}
else {
allocation->initialize_random_device(
options.initialization.seed + seed_shift,
data_distribution);
}
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_host(
data_distribution);
}
else {
allocation->initialize_random_host(
options.initialization.seed + seed_shift,
data_distribution);
}
}
}
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *DeviceContext::allocate_and_initialize_tensor(
Options const &options,
@@ -122,7 +151,6 @@ DeviceAllocation *DeviceContext::allocate_and_initialize_tensor(
data_distribution.set_uniform(1, 4, 0);
break;
case library::NumericTypeID::kF16:
data_distribution.set_uniform(-3, 3, 0);
break;
@@ -168,28 +196,9 @@ DeviceAllocation *DeviceContext::allocate_and_initialize_tensor(
}
}
if (options.initialization.provider == library::Provider::kReferenceDevice) {
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_device(
data_distribution);
}
else {
allocation->initialize_random_device(
options.initialization.seed + seed_shift,
data_distribution);
}
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_host(
data_distribution);
}
else {
allocation->initialize_random_host(
options.initialization.seed + seed_shift,
data_distribution);
}
}
initialize_allocation_with_data_distribution(
options, seed_shift, allocation, data_distribution
);
}
return allocation;
tools/profiler/src/gemm_operation_profiler.cu
+177 -12
View File
@@ -79,6 +79,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
{ArgumentTypeID::kEnumerated, {"runtime_input_datatype_a", "runtime-input-datatype::a"}, "Runtime datatype (e4m3, e5m2, e3m2, e2m3, e2m1)"},
{ArgumentTypeID::kEnumerated, {"runtime_input_datatype_b", "runtime-input-datatype::b"}, "Runtime datatype (e4m3, e5m2, e3m2, e2m3, e2m1)"},
{ArgumentTypeID::kInteger, {"use_pdl", "use-pdl"}, "Use PDL (true, false)"},
{ArgumentTypeID::kEnumerated, {"enable_sm90_mixed_dtype_shuffle_test", "enable-sm90-mixed-dtype-shuffle-test"}, "Enable SM90 mixed input data type kernel shuffle layout test (true, false)"},
{ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
},
{ library::Provider::kCUBLAS}
@@ -211,6 +212,11 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->use_pdl = false;
}
if (!arg_as_bool(this->enable_sm90_mixed_dtype_shuffle_test, "enable_sm90_mixed_dtype_shuffle_test", problem_space, problem)) {
// default value
this->enable_sm90_mixed_dtype_shuffle_test = false;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
// default value
this->split_k_mode = library::SplitKMode::kSerial;
@@ -399,6 +405,7 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "use_pdl", problem_space, library::to_string(use_pdl));
set_argument(result, "enable_sm90_mixed_dtype_shuffle_test", problem_space, library::to_string(enable_sm90_mixed_dtype_shuffle_test));
set_argument(result, "runtime_input_datatype_a", problem_space, library::to_string(runtime_input_datatype_a));
@@ -432,14 +439,26 @@ Status GemmOperationProfiler::initialize_configuration(
Status status = problem_.parse(operation_desc, problem_space, problem);
// Note: this is a temporary workaround
bool is_current_operation_sm90_mixed_dtype_shuffle = (strstr(operation_desc.name, "_shfl") != NULL);
if (is_current_operation_sm90_mixed_dtype_shuffle && (problem_.enable_sm90_mixed_dtype_shuffle_test == false)) {
return Status::kErrorInvalidProblem;
}
if (status != Status::kSuccess) {
return status;
}
const auto device_count = options.device.devices.size();
auto const device_count = options.device.devices.size();
gemm_workspace_.clear();
library::NumericTypeID a_elem = library::get_real_type(operation_desc.A.element);
library::NumericTypeID b_elem = library::get_real_type(operation_desc.B.element);
int a_elem_bits = library::sizeof_bits(a_elem);
int b_elem_bits = library::sizeof_bits(b_elem);
bool is_mixed_input = (a_elem_bits != b_elem_bits);
for (size_t i = 0; i < device_count; ++i) {
cudaSetDevice(options.device.device_id(i));
gemm_workspace_.emplace_back();
@@ -455,7 +474,6 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].configuration.cluster_shape_fallback.m() = int(problem_.cluster_m_fallback);
gemm_workspace_[i].configuration.cluster_shape_fallback.n() = int(problem_.cluster_n_fallback);
gemm_workspace_[i].configuration.cluster_shape_fallback.k() = int(problem_.cluster_k_fallback);
gemm_workspace_[i].configuration.lda = problem_.lda;
gemm_workspace_[i].configuration.ldb = problem_.ldb;
gemm_workspace_[i].configuration.ldc = problem_.ldc;
@@ -501,7 +519,77 @@ Status GemmOperationProfiler::initialize_configuration(
initialize_result_(this->model_result_, options, operation_desc, problem_space);
if (const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments); can_implement != Status::kSuccess) {
if (is_mixed_input)
{
const int options_g = problem_.k;
const int options_l = problem_.batch_count;
const int scale_k = (problem_.k + options_g - 1) / options_g;
// We cannot get the mainloop's ElementScale and ElementZero here,
// use the wide type to allocate a large enough workspace for S and Z.
library::NumericTypeID wide_dtype;
size_t SZ_mat_size = 0;
if (a_elem_bits > b_elem_bits) {
wide_dtype = a_elem;
SZ_mat_size = static_cast<size_t>(problem_.n * scale_k);
}
else {
wide_dtype = b_elem;
SZ_mat_size = static_cast<size_t>(problem_.m * scale_k);
}
gemm_workspace_[i].Scale = device_context.allocate_tensor(
options,
"Scale",
wide_dtype,
library::LayoutTypeID::kRowMajor,
{int(SZ_mat_size), int(options_l)},
{int(options_l)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
gemm_workspace_[i].Zero = device_context.allocate_tensor(
options,
"Zero",
wide_dtype,
library::LayoutTypeID::kRowMajor,
{int(SZ_mat_size), int(options_l)},
{int(options_l)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
// Packed scale is for int4 * fp8, where the original scale is fp8, and
// each scale element will be packed into an Array<fp8, 8> which is 64-bit
gemm_workspace_[i].packed_Scale = device_context.allocate_tensor(
options,
"packed-Scale",
library::NumericTypeID::kU64,
library::LayoutTypeID::kRowMajor,
{int(SZ_mat_size), int(options_l)},
{int(options_l)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
gemm_workspace_[i].arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
// Here is the first touch of the arguments, mark the mixed dtype,
// populate the scale and zero tensors in the following can_implement() call later.
// A and B are not populated at this moment, so do not update the dequantized A or B
gemm_workspace_[i].arguments.is_mixed_dtype = true;
gemm_workspace_[i].arguments.wider_operand = (a_elem_bits > b_elem_bits) ? cutlass::library::Sm90MixedInputWiderOperand::A : cutlass::library::Sm90MixedInputWiderOperand::B;
gemm_workspace_[i].arguments.generate_scale_and_zero = true;
gemm_workspace_[i].arguments.generate_dequantized_AB = false;
gemm_workspace_[i].arguments.dequantized_AB_ready = (bool *) malloc(sizeof(bool));
gemm_workspace_[i].arguments.dequantized_AB_ready[0] = false;
gemm_workspace_[i].arguments.Scale = gemm_workspace_[i].Scale->data();
gemm_workspace_[i].arguments.Zero = gemm_workspace_[i].Zero->data();
gemm_workspace_[i].arguments.packed_Scale = gemm_workspace_[i].packed_Scale->data();
} // End of "if (is_mixed_input)"
const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments);
if (can_implement != Status::kSuccess) {
return can_implement;
}
}
@@ -693,6 +781,56 @@ Status GemmOperationProfiler::initialize_workspace(
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
// Encoded tensor has the same shape and data type of the narrow data type tensor
if (gemm_workspace_[i].arguments.wider_operand == cutlass::library::Sm90MixedInputWiderOperand::A) {
gemm_workspace_[i].dequantized_AB = device_context.allocate_tensor(
options,
"dequantized-B",
operation_desc.A.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
gemm_workspace_[i].encoded_AB = device_context.allocate_tensor(
options,
"encoded-B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
}
else {
gemm_workspace_[i].dequantized_AB = device_context.allocate_tensor(
options,
"dequantized-A",
operation_desc.B.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
gemm_workspace_[i].encoded_AB = device_context.allocate_tensor(
options,
"encoded-A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
}
}
}
if (options.execution_mode != ExecutionMode::kDryRun) {
@@ -712,7 +850,7 @@ Status GemmOperationProfiler::initialize_workspace(
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
/* Query device SM count to pass onto the kernel as an argument, where needed */
gemm_workspace_[i].arguments.sm_count = options.device.properties[0].multiProcessorCount;
gemm_workspace_[i].arguments.sm_count = options.device.properties[i].multiProcessorCount;
gemm_workspace_[i].arguments.device_index = static_cast<int>(i);
}
}
@@ -836,6 +974,17 @@ bool GemmOperationProfiler::verify_cutlass(
gemm_workspace_[i].arguments.batch_stride_C = gemm_workspace_[i].C->batch_stride();
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
// Scale and zero already generated in initialize_configuration(),
// A and B already generated in initialize_workspace(), signal
// GemmUniversal3xOperation::update_arguments_() (trigger by underlying_operation->run())
// to generate the dequantized matrix for verification
gemm_workspace_[i].arguments.generate_scale_and_zero = false;
gemm_workspace_[i].arguments.generate_dequantized_AB = true;
gemm_workspace_[i].arguments.dequantized_AB = gemm_workspace_[i].dequantized_AB->data();
gemm_workspace_[i].arguments.encoded_AB = gemm_workspace_[i].encoded_AB->data();
}
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[i].arguments.D = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].arguments.alpha = problem_.alpha_one.data();
@@ -1133,7 +1282,6 @@ bool GemmOperationProfiler::verify_with_reference_(
//
// Initialize state
//
for (auto provider : options.verification.providers) {
// Skip providers that are not enabled
@@ -1149,6 +1297,21 @@ bool GemmOperationProfiler::verify_with_reference_(
void *ptr_C = gemm_workspace_[i].C->data();
void *ptr_D = gemm_workspace_[i].Reference->data();
cutlass::library::NumericTypeID element_A_for_reference = element_A;
cutlass::library::NumericTypeID element_B_for_reference = element_B;
if (gemm_workspace_[i].arguments.is_mixed_dtype && gemm_workspace_[i].arguments.dequantized_AB_ready[0]) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
if (gemm_workspace_[i].arguments.wider_operand == cutlass::library::Sm90MixedInputWiderOperand::A) {
ptr_B = gemm_workspace_[i].dequantized_AB->data();
element_B_for_reference = element_A;
}
else {
ptr_A = gemm_workspace_[i].dequantized_AB->data();
element_A_for_reference = element_B;
}
}
// To support the host-side reference, conditionally allocate and
// copy tensors to host memory.
std::vector<uint8_t> host_data_A;
@@ -1200,13 +1363,13 @@ bool GemmOperationProfiler::verify_with_reference_(
problem_.alpha.data(),
element_A,
element_A_for_reference,
gemm_desc.A.layout,
gemm_desc.transform_A,
ptr_A,
int(gemm_workspace_[i].configuration.lda),
element_B,
element_B_for_reference,
gemm_desc.B.layout,
gemm_desc.transform_B,
ptr_B,
@@ -1349,6 +1512,13 @@ Status GemmOperationProfiler::profile_cutlass_(
gemm_workspace_[dev_id].arguments.C = gemm_workspace_[dev_id].C->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
if (gemm_workspace_[dev_id].arguments.is_mixed_dtype) {
// Scale, zero, and dequantized tensors are already generated in
// verify_cutlass(), no need to re-generate them in profiling
gemm_workspace_[dev_id].arguments.generate_scale_and_zero = false;
gemm_workspace_[dev_id].arguments.generate_dequantized_AB = false;
}
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].device_workspace.data();
@@ -1383,11 +1553,6 @@ Status GemmOperationProfiler::profile_cutlass_(
return Status::kSuccess;
};
if (options.device.devices.size() == 1) {
auto func = [&](cudaStream_t stream, int iteration) { return launch_gemm(0, stream, iteration); };
return profile_kernel_(result, options, func, gemm_workspace_[0].stream);
}
std::vector<cudaStream_t> streams(gemm_workspace_.size());
for (size_t i = 0; i < streams.size(); i++) {
streams[i] = gemm_workspace_[i].stream;
tools/profiler/src/grouped_gemm_operation_profiler.cu
+1034
View File
File diff suppressed because it is too large Load Diff
tools/profiler/src/operation_profiler.cu
+82 -39
View File
@@ -57,16 +57,6 @@
///////////////////////////////////////////////////////////////////////////////////////////////////
#define CUDA_CHECK(call) \
do { \
cudaError_t err = call; \
if (err != cudaSuccess) { \
std::cerr << "CUDA error at " << __FILE__ << ":" << __LINE__ << " code=" << err << " \"" \
<< cudaGetErrorString(err) << "\"\n"; \
return Status::kErrorInternal; \
} \
} while (0)
namespace cutlass {
namespace profiler {
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -304,42 +294,43 @@ std::ostream& operator<<(std::ostream& out, library::Provider provider) {
return out;
}
std::ostream& operator<<(std::ostream& out, library::OperationKind provider) {
if (provider == library::OperationKind::kGemm) {
std::ostream& operator<<(std::ostream& out, library::OperationKind op_kind) {
if (op_kind == library::OperationKind::kGemm) {
out << "kGemm";
}
else if (provider == library::OperationKind::kBlockScaledGemm) {
else if (op_kind == library::OperationKind::kBlockScaledGemm) {
out << "kBlockScaledGemm";
}
else if (provider == library::OperationKind::kRankK) {
else if (op_kind == library::OperationKind::kRankK) {
out << "kRankK";
}
else if (provider == library::OperationKind::kRank2K) {
else if (op_kind == library::OperationKind::kRank2K) {
out << "kRank2K";
}
else if (provider == library::OperationKind::kTrmm) {
else if (op_kind == library::OperationKind::kTrmm) {
out << "kTrmm";
}
else if (provider == library::OperationKind::kSymm) {
else if (op_kind == library::OperationKind::kSymm) {
out << "kSymm";
}
else if (provider == library::OperationKind::kConv2d) {
else if (op_kind == library::OperationKind::kConv2d) {
out << "kConv2d";
}
else if (provider == library::OperationKind::kConv3d) {
else if (op_kind == library::OperationKind::kConv3d) {
out << "kConv3d";
}
else if (provider == library::OperationKind::kEqGemm) {
else if (op_kind == library::OperationKind::kEqGemm) {
out << "kEqGemm";
}
else if (provider == library::OperationKind::kSparseGemm) {
else if (op_kind == library::OperationKind::kSparseGemm) {
out << "kSparseGemm";
}
else if (provider == library::OperationKind::kReduction) {
else if (op_kind == library::OperationKind::kReduction) {
out << "kReduction";
}
else if (op_kind == library::OperationKind::kGroupedGemm) {
out << "kGroupedGemm";
}
else {
out << "kInvalid";
}
@@ -660,6 +651,11 @@ void OperationProfiler::save_workspace(
DeviceAllocation *allocation = named_allocation.second;
if (allocation->layout() == library::LayoutTypeID::kUnknown) {
continue; // write_tensor not set up to handle DeviceAllocations initialized using
// allocate_block()
}
std::stringstream filename;
filename << desc.name << "_" << library::to_string(provider) << "_";
@@ -736,15 +732,20 @@ Status predict_iters(
/// CUDA graphs allows you to record the launch of large numbers of kernels without
/// blocking and therefore avoids a deadlock which happens if you try to enqueue too
/// many kernels behind the spinloop kernel.
Status OperationProfiler::profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(int, cudaStream_t, int)> &func,
const std::vector<cudaStream_t> &streams) {
Status OperationProfiler::profile_kernel_w_cuda_graphs_(
PerformanceResult& result,
Options const& options,
std::function<Status(int, cudaStream_t, int)> const& func,
std::vector<cudaStream_t> const& streams) {
auto dev_count = streams.size();
cuda::atomic<bool> *release;
CUDA_CHECK(cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable));
release->store(false, cuda::memory_order_release);
if (dev_count > 1) {
CUDA_CHECK(cudaHostAlloc(&release, sizeof(*release), cudaHostAllocPortable));
release->store(false, cuda::memory_order_release);
}
std::vector<GpuTimer> timer;
for (size_t i = 0; i < dev_count; ++i) {
@@ -774,9 +775,11 @@ Status OperationProfiler::profile_kernel_(
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
CUDA_CHECK(cudaStreamBeginCapture(streams[i], cudaStreamCaptureModeGlobal));
// Halt execution until all GPUs are ready to precede.
// It allows the CPU to trigger the GPUs all start at the same time.
delay<<<1, 1, 0, streams[i]>>>(release);
if (dev_count > 1) {
// Halt execution until all GPUs are ready to precede.
// It allows the CPU to trigger the GPUs all start at the same time.
delay<<<1, 1, 0, streams[i]>>>(release);
}
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
Status status = func(i, streams[i], iteration);
if (status != Status::kSuccess) {
@@ -803,8 +806,10 @@ Status OperationProfiler::profile_kernel_(
CUDA_CHECK(cudaGraphLaunch(graphExecs[i], streams[i]));
}
// release the enqueued kernels
release->store(true, cuda::memory_order_release);
if (dev_count > 1) {
// release the enqueued kernels
release->store(true, cuda::memory_order_release);
}
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
@@ -819,7 +824,9 @@ Status OperationProfiler::profile_kernel_(
}
result.runtime /= static_cast<double>(dev_count);
CUDA_CHECK(cudaFreeHost(release));
if (dev_count > 1) {
CUDA_CHECK(cudaFreeHost(release));
}
for (size_t i = 0; i < dev_count; ++i) {
CUDA_CHECK(cudaSetDevice(options.device.device_id(i)));
@@ -835,11 +842,47 @@ Status OperationProfiler::profile_kernel_(
return Status::kSuccess;
}
/// Method to profile GPU execution time of a kernel launched in func
Status OperationProfiler::profile_kernel_(
PerformanceResult &result,
Options const &options,
const std::function<Status(cudaStream_t, int)> &func,
const std::function<Status(int, cudaStream_t, int)> &func,
const std::vector<cudaStream_t> &streams) {
if (options.profiling.use_cuda_graphs) {
return profile_kernel_w_cuda_graphs_(result, options, func, streams);
}
else if (streams.size() == 1) {
auto single_device_func = [&](cudaStream_t stream, int iteration) {
return func(0, stream, iteration);
};
return profile_kernel_no_cuda_graphs_(result, options, single_device_func, streams[0]);
}
return Status::kErrorNotSupported;
}
/// Method to profile GPU execution time of a kernel launched in func
Status OperationProfiler::profile_kernel_(
PerformanceResult& result,
Options const& options,
std::function<Status(cudaStream_t, int)> const& func,
cudaStream_t stream) {
if (options.profiling.use_cuda_graphs) {
auto graph_func = [&](int dev_id, cudaStream_t stream, int iteration) {
return func(stream, iteration);
};
return profile_kernel_w_cuda_graphs_(result, options, graph_func, {stream});
} else {
return profile_kernel_no_cuda_graphs_(result, options, func, stream);
}
return Status::kSuccess;
}
/// Method to profile GPU execution time of a kernel launched in func
Status OperationProfiler::profile_kernel_no_cuda_graphs_(
PerformanceResult& result,
Options const& options,
std::function<Status(cudaStream_t, int)> const& func,
cudaStream_t stream) {
GpuTimer timer;
tools/profiler/src/options.cu
+1
View File
@@ -477,6 +477,7 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("profiling-enabled", enabled, true);
cmdline.get_cmd_line_argument("profiling-duration", duration, 10);
cmdline.get_cmd_line_argument("min-iterations", min_iterations, 10);
cmdline.get_cmd_line_argument("use-cuda-graphs", use_cuda_graphs, false);
if (cmdline.check_cmd_line_flag("providers")) {
tools/profiler/src/problem_space.cpp
+28
View File
@@ -1203,6 +1203,34 @@ bool arg_as_scalar(
return arg_as_scalar(bytes, numeric_type, value_ptr);
}
/// Returns a copy of the string passed to the argument.
/// (kScalar arguments are stored as strings).
bool arg_as_string(
std::string& arg,
char const* name,
ProblemSpace const& problem_space,
ProblemSpace::Problem const& problem) {
size_t idx = problem_space.argument_index(name);
KernelArgument::Value const* value_ptr = problem.at(idx).get();
if (value_ptr->not_null) {
if (value_ptr->argument->description->type == ArgumentTypeID::kScalar) {
std::string const& str_value =
static_cast<ScalarArgument::ScalarValue const*>(value_ptr)->value;
arg = std::string(str_value);
}
else {
throw std::runtime_error(
"arg_as_string() - illegal cast. Problem space argument must be scalar");
}
return true;
}
return false;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns true if a tensor description satisfies a `tensor` value