leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/conv2d_operation_profiler.cu
+124 -105
View File
@@ -51,10 +51,10 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
Conv2dOperationProfiler::Conv2dOperationProfiler(Options const &options):
Conv2dOperationProfiler::Conv2dOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kConv2d,
library::OperationKind::kConv2d,
{
{ArgumentTypeID::kEnumerated, {"conv_kind"}, "Convolutional operator (fprop, dgrad, wgrad)"},
{ArgumentTypeID::kInteger, {"n", "input_n"}, "Input N dimension of the Conv2d problem space"},
@@ -165,13 +165,13 @@ int64_t Conv2dOperationProfiler::Conv2dProblem::flops(
int64_t flops_mainloop_ = int64_t(mnk.m()) * mnk.n() * mnk.k() * 2;
int64_t flops_epilogue_ = int64_t(mnk.m()) * int64_t(mnk.n()) * 2;
// Adjust mainloop flop for dgrad strided
if (operation_desc.conv_kind == library::ConvKind::kDgrad) {
flops_mainloop_ = flops_mainloop_ / (stride_h * stride_w);
}
int64_t flops_total_ = flops_mainloop_ + flops_epilogue_;
//complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
case library::MathOperationID::kMultiplyAddComplex:
@@ -188,14 +188,14 @@ int64_t Conv2dOperationProfiler::Conv2dProblem::flops(
/// Extracts the problem dimensions
Status Conv2dOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::ConvDescription const &operation_desc =
library::ConvDescription const &operation_desc =
static_cast<library::ConvDescription const &>(operation->description());
if (!arg_as_int(problem_.n, "n", problem_space, problem)) {
@@ -207,7 +207,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
// default value
problem_.h = 16;
}
if (!arg_as_int(problem_.w, "w", problem_space, problem)) {
// default value
problem_.w = 16;
@@ -227,7 +227,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
// default value
problem_.r = 3;
}
if (!arg_as_int(problem_.s, "s", problem_space, problem)) {
// default value
problem_.s = 3;
@@ -280,14 +280,14 @@ Status Conv2dOperationProfiler::initialize_configuration(
// cutlass profiler sets p and q which are cuDNN compliant. //
// //
////////////////////////////////////////////////////////////////////////////////////////
// set convolution output p
// set convolution output p
if (!arg_as_int(problem_.p, "p", problem_space, problem)) {
// default value (set using cudnn formula for output height, when p is not provided)
problem_.p = (
problem_.h +
2 * problem_.pad_h -
problem_.h +
2 * problem_.pad_h -
((problem_.r - 1) * problem_.dilation_h + 1)
) / (problem_.stride_h)
) / (problem_.stride_h)
+ 1;
}
@@ -295,10 +295,10 @@ Status Conv2dOperationProfiler::initialize_configuration(
if (!arg_as_int(problem_.q, "q", problem_space, problem)) {
// default value (set using cudnn formula for output width, when q is not provided)
problem_.q = (
problem_.w +
2 * problem_.pad_w -
problem_.w +
2 * problem_.pad_w -
((problem_.s - 1) * problem_.dilation_w + 1)
) / (problem_.stride_w)
) / (problem_.stride_w)
+ 1;
}
/////////////////////////////////////////////////////////////////////////////////////////
@@ -313,7 +313,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
// default value
problem_.split_k_slices = 1;
}
if (!arg_as_ConvModeID(problem_.conv_mode, "conv_mode", problem_space, problem)) {
// default value
problem_.conv_mode = library::ConvModeID::kCrossCorrelation;
@@ -345,24 +345,24 @@ Status Conv2dOperationProfiler::initialize_configuration(
}
if (!arg_as_scalar(
problem_.alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem_.alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(problem_.alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
problem_.beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem_.beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(problem_.beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
@@ -389,7 +389,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
int(problem_.split_k_slices),
int(problem_.groups)
);
conv_workspace_.configuration.split_k_mode = static_cast<conv::SplitKMode>(static_cast<int>(problem_.split_k_mode));
conv_workspace_.set_stride_vector(
@@ -420,7 +420,7 @@ Status Conv2dOperationProfiler::initialize_configuration(
/// Initializes the performance result
void Conv2dOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::ConvDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -432,15 +432,15 @@ void Conv2dOperationProfiler::initialize_result_(
result.arguments.resize(problem_space.rank());
set_argument(result, "Activation", problem_space,
std::string(library::to_string(operation_desc.activation().element))
std::string(library::to_string(operation_desc.activation().element))
+ ":" + library::to_string(operation_desc.activation().layout));
set_argument(result, "Filter", problem_space,
std::string(library::to_string(operation_desc.filter().element))
std::string(library::to_string(operation_desc.filter().element))
+ ":" + library::to_string(operation_desc.filter().layout));
set_argument(result, "Output", problem_space,
std::string(library::to_string(operation_desc.output().element))
std::string(library::to_string(operation_desc.output().element))
+ ":" + library::to_string(operation_desc.output().layout));
set_argument(result, "conv_kind", problem_space, library::to_string(operation_desc.conv_kind));
@@ -455,7 +455,7 @@ void Conv2dOperationProfiler::initialize_result_(
set_argument(result, "k", problem_space, problem_.k);
set_argument(result, "r", problem_space, problem_.r);
set_argument(result, "s", problem_space, problem_.s);
set_argument(result, "p", problem_space, problem_.p);
set_argument(result, "q", problem_space, problem_.q);
@@ -470,11 +470,11 @@ void Conv2dOperationProfiler::initialize_result_(
set_argument(result, "dilation_h", problem_space, problem_.dilation_h);
set_argument(result, "dilation_w", problem_space, problem_.dilation_w);
set_argument(result, "split_k_mode", problem_space,
set_argument(result, "split_k_mode", problem_space,
std::string(library::to_string(problem_.split_k_mode)));
set_argument(result, "split_k_slices", problem_space, problem_.split_k_slices);
set_argument(result, "conv_mode", problem_space,
set_argument(result, "conv_mode", problem_space,
std::string(library::to_string(problem_.conv_mode)));
set_argument(result, "alpha", problem_space,
@@ -483,19 +483,19 @@ void Conv2dOperationProfiler::initialize_result_(
set_argument(result, "beta", problem_space,
library::lexical_cast(problem_.beta, operation_desc.element_epilogue));
set_argument(result, "eq_gemm_provider", problem_space,
set_argument(result, "eq_gemm_provider", problem_space,
std::string(library::to_string(problem_.eq_gemm_provider)));
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
// Bytes of activation, filter, and output tensors
int64_t activation_bytes = int64_t(library::sizeof_bits(operation_desc.activation().element) / 8) *
int64_t activation_bytes = int64_t(library::sizeof_bits(operation_desc.activation().element) / 8) *
conv_workspace_.configuration.problem_size.activation_size();
int64_t filter_bytes = int64_t(library::sizeof_bits(operation_desc.filter().element) / 8) *
int64_t filter_bytes = int64_t(library::sizeof_bits(operation_desc.filter().element) / 8) *
conv_workspace_.configuration.problem_size.filter_size();
int64_t output_bytes = int64_t(library::sizeof_bits(operation_desc.output().element) / 8) *
int64_t output_bytes = int64_t(library::sizeof_bits(operation_desc.output().element) / 8) *
conv_workspace_.configuration.problem_size.output_size();
// Bytes of activation, filter, and output tensors
@@ -511,14 +511,14 @@ void Conv2dOperationProfiler::initialize_result_(
/// Initialize reduction problem dimensions and library::Operation
bool Conv2dOperationProfiler::initialize_reduction_configuration_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::ConvDescription const &conv_desc =
library::ConvDescription const &conv_desc =
static_cast<library::ConvDescription const &>(operation->description());
library::ConvKind const &conv_kind = conv_desc.conv_kind;
@@ -545,14 +545,14 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
conv_workspace_.reduction_configuration.ldd =
conv_workspace_.configuration.stride_c[tensor_c_stride_idx];
// find reduction operation
// find reduction operation
library::ReductionFunctionalKey reduction_key(
library::Provider::kCUTLASS,
conv_desc.tile_description.math_instruction.element_accumulator, // element workspace
conv_desc.tile_description.math_instruction.element_accumulator, // element workspace
conv_desc.tile_description.math_instruction.element_accumulator, // element accumulator
conv_desc.C.element, // element output
conv_desc.element_epilogue // element compute
);
);
#if 0// debug print to check which reduction instance is selected
std::cout << reduction_key << "\n";
@@ -562,7 +562,7 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
if(reduction_it == Singleton::get().operation_table.reduction_operations.end()) {
return false;
}
}
// initialize reduction operation required for parallel split-k conv2d operator
reduction_op_ = reduction_it->second;
@@ -574,13 +574,24 @@ bool Conv2dOperationProfiler::initialize_reduction_configuration_(
/// Initializes workspace
Status Conv2dOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
// initialize conv2d underlying operation to handle parallel reduction
library::Operation const* underlying_operation = operation;
@@ -590,15 +601,15 @@ Status Conv2dOperationProfiler::initialize_workspace(
}
}
library::ConvDescription const &operation_desc =
library::ConvDescription const &operation_desc =
static_cast<library::ConvDescription const &>(underlying_operation->description());
// Compute the number of copies of the problem to avoid L2 camping.
if (!options.profiling.workspace_count) {
int64_t bytes = problem_.bytes(operation_desc);
if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
if (bytes < 3 * int64_t(options.device.properties[0].l2CacheSize)) {
conv_workspace_.problem_count =
1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
1 + int((3 * int64_t(options.device.properties[0].l2CacheSize)) / bytes);
}
else {
conv_workspace_.problem_count = 1;
@@ -611,7 +622,7 @@ Status Conv2dOperationProfiler::initialize_workspace(
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
conv_workspace_.A = device_context.allocate_tensor(
conv_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -619,10 +630,11 @@ Status Conv2dOperationProfiler::initialize_workspace(
problem_.extent_a(operation_desc.conv_kind),
conv_workspace_.configuration.stride_a,
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
conv_workspace_.B = device_context.allocate_tensor(
conv_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -630,12 +642,13 @@ Status Conv2dOperationProfiler::initialize_workspace(
problem_.extent_b(operation_desc.conv_kind),
conv_workspace_.configuration.stride_b,
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
if(problem_.groups == problem_.c && problem_.groups == problem_.k){
// Depthwise direct conv kernel needs reorder the filter.
conv_workspace_.reordered_B = device_context.allocate_tensor(
conv_workspace_.reordered_B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -643,11 +656,12 @@ Status Conv2dOperationProfiler::initialize_workspace(
problem_.extent_b(operation_desc.conv_kind),
conv_workspace_.configuration.stride_b,
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
}
conv_workspace_.C = device_context.allocate_tensor(
conv_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -655,25 +669,30 @@ Status Conv2dOperationProfiler::initialize_workspace(
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.configuration.stride_c,
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
conv_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.configuration.stride_c,
conv_workspace_.problem_count
conv_workspace_.problem_count,
0 // device_index
);
conv_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.configuration.stride_c,
conv_workspace_.problem_count
conv_workspace_.problem_count,
0 // device_index
);
}
@@ -706,10 +725,10 @@ Status Conv2dOperationProfiler::initialize_workspace(
conv_workspace_.reduction_host_workspace.resize(workspace_size, 0);
status = reduction_op_->initialize(
&conv_workspace_.reduction_configuration,
conv_workspace_.reduction_host_workspace.data(),
&conv_workspace_.reduction_configuration,
conv_workspace_.reduction_host_workspace.data(),
nullptr);
if (status != Status::kSuccess) {
return status;
}
@@ -736,7 +755,7 @@ Status Conv2dOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool Conv2dOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -769,7 +788,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
}
conv_workspace_.Computed->copy_from_device(conv_workspace_.C->data());
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
// update library::ConvArguments for parallel split-k reduction
conv_workspace_.arguments.D = conv_workspace_.device_workspace.data();
@@ -799,9 +818,9 @@ bool Conv2dOperationProfiler::verify_cutlass(
}
#if 0
std::cout << "profiling : " << std::endl
<< "conv2d : " << operation->description().name << std::endl
<< "underlying conv2d : " << underlying_operation->description().name << std::endl
std::cout << "profiling : " << std::endl
<< "conv2d : " << operation->description().name << std::endl
<< "underlying conv2d : " << underlying_operation->description().name << std::endl
<< "reduction : " << reduction_op_->description().name << std::endl;
#endif
@@ -818,7 +837,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
results_.back().status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
conv_workspace_.reduction_host_workspace.data(),
@@ -840,7 +859,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
// CUTLASS op ran the but not yet verified against any verification provider
results_.back().disposition = Disposition::kNotVerified;
//
// Run verification providers
//
@@ -856,7 +875,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
Status status = cudnn_satisfies(conv_desc, conv_workspace_.configuration);
// Initialize reference data to the source data
// Initialize reference data to the source data
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
if (status == Status::kSuccess) {
@@ -884,7 +903,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
// Run verification device reference
if (options.verification.provider_enabled(library::Provider::kReferenceDevice)) {
// Restore reference data back to initial source data
// Restore reference data back to initial source data
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
verify_with_device_reference_(
@@ -893,13 +912,13 @@ bool Conv2dOperationProfiler::verify_cutlass(
device_context,
operation,
problem_space,
problem);
problem);
}
// Run verification host reference
if (options.verification.provider_enabled(library::Provider::kReferenceHost)) {
// Restore reference data back to initial source data
// Restore reference data back to initial source data
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
verify_with_host_reference_(
@@ -908,10 +927,10 @@ bool Conv2dOperationProfiler::verify_cutlass(
device_context,
operation,
problem_space,
problem);
problem);
}
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -936,7 +955,7 @@ bool Conv2dOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against host reference
bool Conv2dOperationProfiler::verify_with_host_reference_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -954,14 +973,14 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
library::ConvFunctionalKey conv2d_key(
library::Provider::kReferenceHost,
conv_desc.conv_kind,
conv_desc.conv_kind,
conv_desc.A.element,
conv_desc.A.layout,
conv_desc.B.element,
conv_desc.B.layout,
conv_desc.C.element,
conv_desc.C.layout,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.element_epilogue);
#if 0 // debug print to check which host reference instance is selected
@@ -974,12 +993,12 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
return true;
}
}
// conv2d host reference minimum cc is 0 (CPU) and no iterator algorithm
library::ConvPreferenceKey preference_key(0, library::IteratorAlgorithmID::kNone);
auto cc_it = operators_it->second.find(preference_key);
if(cc_it == operators_it->second.end()) {
results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
return true;
@@ -1052,9 +1071,9 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kReferenceHost] == Disposition::kIncorrect) {
save_workspace(
device_context,
options,
@@ -1070,7 +1089,7 @@ bool Conv2dOperationProfiler::verify_with_host_reference_(
/// Verifies CUTLASS against host reference
bool Conv2dOperationProfiler::verify_with_device_reference_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -1088,14 +1107,14 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
library::ConvFunctionalKey conv2d_key(
library::Provider::kReferenceDevice,
conv_desc.conv_kind,
conv_desc.conv_kind,
conv_desc.A.element,
conv_desc.A.layout,
conv_desc.B.element,
conv_desc.B.layout,
conv_desc.C.element,
conv_desc.C.layout,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.element_epilogue);
auto operators_it = Singleton::get().operation_table.conv2d_operations.find(conv2d_key);
@@ -1105,12 +1124,12 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
results_.back().verification_map[library::Provider::kReferenceDevice] = Disposition::kNotRun;
return true;
}
}
// conv2d device reference minimum cc is 50 and no iterator algorithm
library::ConvPreferenceKey preference_key(50, library::IteratorAlgorithmID::kNone);
auto cc_it = operators_it->second.find(preference_key);
if(cc_it == operators_it->second.end()) {
results_.back().verification_map[library::Provider::kReferenceDevice] = Disposition::kNotRun;
@@ -1119,7 +1138,7 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
// device reference has only one instances in Conv2dOperationVectorMap
library::Operation const *reference_op = cc_it->second[0];
//
// Initialize device reference operation
//
@@ -1166,9 +1185,9 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kReferenceDevice] == Disposition::kIncorrect) {
save_workspace(
device_context,
options,
@@ -1183,14 +1202,14 @@ bool Conv2dOperationProfiler::verify_with_device_reference_(
/// Measures performance results
bool Conv2dOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing Conv2d arguments
@@ -1242,7 +1261,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
GpuTimer timer;
// initialize conv2d underlying operation to handle parallel reduction
library::Operation const* underlying_operation = operation;
library::Operation const* underlying_operation = operation;
library::ConvArguments *conv_arguments = static_cast<library::ConvArguments *>(arguments);
@@ -1274,7 +1293,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
conv_arguments->B = conv_workspace_.B->batch_data(problem_idx);
conv_arguments->C = conv_workspace_.C->batch_data(problem_idx);
conv_arguments->D = conv_workspace_.Computed->batch_data(problem_idx);
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
// update library::ConvArguments for parallel split-k reduction
conv_arguments->D = conv_workspace_.device_workspace.data();
@@ -1304,7 +1323,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
return status;
}
}
//
// Initialize GPU timer
//
@@ -1319,7 +1338,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
// Setup rotating workspace
int problem_idx = (iteration % conv_workspace_.problem_count);
@@ -1345,7 +1364,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
device_workspace);
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
@@ -1367,7 +1386,7 @@ Status Conv2dOperationProfiler::profile_cutlass_(
//
// Update performance result
//
runtime = timer.duration(iteration);
return status;
@@ -1378,13 +1397,13 @@ Status Conv2dOperationProfiler::profile_cutlass_(
/// Verifies CUTLASS against cudnn reference
bool Conv2dOperationProfiler::verify_with_cudnn_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
auto &conv_desc = static_cast<library::ConvDescription const &>(operation->description());
//
@@ -1395,7 +1414,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
cudnnStatus_t status = handle.get_cudnn_create_status();
if (status != CUDNN_STATUS_SUCCESS) {
results_.back().verification_map[library::Provider::kCUDNN] = get_cutlass_disposition(status);
return true;
}
@@ -1411,7 +1430,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
conv_workspace_.arguments.alpha = problem_.alpha.data();
conv_workspace_.arguments.beta = problem_.beta.data();
conv_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
// cuDNN does not support four tensor arguments, so we copy the tensor C data into
// tensor D.
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
@@ -1423,8 +1442,8 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
// Construct dispatcher to cudnn operator
//
detail::cudnnConvDispatcher conv_op(
conv_desc,
detail::cudnnConvDispatcher conv_op(
conv_desc,
conv_workspace_.configuration,
conv_workspace_.arguments,
handle
@@ -1462,7 +1481,7 @@ bool Conv2dOperationProfiler::verify_with_cudnn_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUDNN] == Disposition::kIncorrect) {
save_workspace(
tools/profiler/src/conv3d_operation_profiler.cu
+113 -95
View File
@@ -52,10 +52,10 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
Conv3dOperationProfiler::Conv3dOperationProfiler(Options const &options):
Conv3dOperationProfiler::Conv3dOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kConv3d,
library::OperationKind::kConv3d,
{
{ArgumentTypeID::kEnumerated, {"conv_kind"}, "Convolutional operator (fprop, dgrad, wgrad)"},
{ArgumentTypeID::kInteger, {"n", "input_n"}, "Input N dimension of the Conv3d problem space"},
@@ -170,7 +170,7 @@ int64_t Conv3dOperationProfiler::Conv3dProblem::flops(
int64_t flops_mainloop_ = int64_t(mnk.m()) * mnk.n() * mnk.k() * 2;
int64_t flops_epilogue_ = int64_t(mnk.m()) * int64_t(mnk.n()) * 2;
// Adjust mainloop flop for dgrad strided
if (operation_desc.conv_kind == library::ConvKind::kDgrad) {
flops_mainloop_ = flops_mainloop_ / ( stride_d * stride_h * stride_w);
@@ -183,14 +183,14 @@ int64_t Conv3dOperationProfiler::Conv3dProblem::flops(
/// Extracts the problem dimensions
Status Conv3dOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::ConvDescription const &operation_desc =
library::ConvDescription const &operation_desc =
static_cast<library::ConvDescription const &>(operation->description());
if (!arg_as_int(problem_.n, "n", problem_space, problem)) {
@@ -207,7 +207,7 @@ Status Conv3dOperationProfiler::initialize_configuration(
// default value
problem_.h = 14;
}
if (!arg_as_int(problem_.w, "w", problem_space, problem)) {
// default value
problem_.w = 14;
@@ -232,7 +232,7 @@ Status Conv3dOperationProfiler::initialize_configuration(
// default value
problem_.r = 3;
}
if (!arg_as_int(problem_.s, "s", problem_space, problem)) {
// default value
problem_.s = 3;
@@ -294,25 +294,25 @@ Status Conv3dOperationProfiler::initialize_configuration(
// cutlass profiler sets p and q which are cuDNN compliant. //
// //
////////////////////////////////////////////////////////////////////////////////////////
// set convolution output z
// set convolution output z
if (!arg_as_int(problem_.z, "z", problem_space, problem)) {
// default value (set using cudnn formula for output height, when p is not provided)
problem_.z = (
problem_.d +
2 * problem_.pad_d -
problem_.d +
2 * problem_.pad_d -
((problem_.t - 1) * problem_.dilation_d + 1)
) / (problem_.stride_d)
) / (problem_.stride_d)
+ 1;
}
// set convolution output p
// set convolution output p
if (!arg_as_int(problem_.p, "p", problem_space, problem)) {
// default value (set using cudnn formula for output height, when p is not provided)
problem_.p = (
problem_.h +
2 * problem_.pad_h -
problem_.h +
2 * problem_.pad_h -
((problem_.r - 1) * problem_.dilation_h + 1)
) / (problem_.stride_h)
) / (problem_.stride_h)
+ 1;
}
@@ -320,10 +320,10 @@ Status Conv3dOperationProfiler::initialize_configuration(
if (!arg_as_int(problem_.q, "q", problem_space, problem)) {
// default value (set using cudnn formula for output width, when q is not provided)
problem_.q = (
problem_.w +
2 * problem_.pad_w -
problem_.w +
2 * problem_.pad_w -
((problem_.s - 1) * problem_.dilation_w + 1)
) / (problem_.stride_w)
) / (problem_.stride_w)
+ 1;
}
/////////////////////////////////////////////////////////////////////////////////////////
@@ -338,7 +338,7 @@ Status Conv3dOperationProfiler::initialize_configuration(
// default value
problem_.split_k_slices = 1;
}
if (!arg_as_ConvModeID(problem_.conv_mode, "conv_mode", problem_space, problem)) {
// default value
problem_.conv_mode = library::ConvModeID::kCrossCorrelation;
@@ -370,24 +370,24 @@ Status Conv3dOperationProfiler::initialize_configuration(
}
if (!arg_as_scalar(
problem_.alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem_.alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(problem_.alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
problem_.beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem_.beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(problem_.beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
@@ -420,25 +420,25 @@ Status Conv3dOperationProfiler::initialize_configuration(
int(problem_.split_k_slices),
1 // groups
);
conv_workspace_.configuration.split_k_mode = static_cast<conv::SplitKMode>(static_cast<int>(problem_.split_k_mode));
conv_workspace_.configuration.layout_activations.stride() = make_Coord(
int(problem_.c),
int(problem_.c),
int(problem_.w) * int(problem_.c),
int(problem_.h) * int(problem_.w) * int(problem_.c),
int(problem_.d) * int(problem_.h) * int(problem_.w) * int(problem_.c)
);
conv_workspace_.configuration.layout_filters.stride() = make_Coord(
int(problem_.c),
int(problem_.c),
int(problem_.s) * int(problem_.c),
int(problem_.r) * int(problem_.s) * int(problem_.c),
int(problem_.t) * int(problem_.r) * int(problem_.s) * int(problem_.c)
);
conv_workspace_.configuration.layout_output.stride() = make_Coord(
int(problem_.k),
int(problem_.k),
int(problem_.q) * int(problem_.k),
int(problem_.q) * int(problem_.p) * int(problem_.k),
int(problem_.z) * int(problem_.q) * int(problem_.p) * int(problem_.k)
@@ -469,7 +469,7 @@ Status Conv3dOperationProfiler::initialize_configuration(
/// Initializes the performance result
void Conv3dOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::ConvDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -481,15 +481,15 @@ void Conv3dOperationProfiler::initialize_result_(
result.arguments.resize(problem_space.rank());
set_argument(result, "Activation", problem_space,
std::string(library::to_string(operation_desc.activation().element))
std::string(library::to_string(operation_desc.activation().element))
+ ":" + library::to_string(operation_desc.activation().layout));
set_argument(result, "Filter", problem_space,
std::string(library::to_string(operation_desc.filter().element))
std::string(library::to_string(operation_desc.filter().element))
+ ":" + library::to_string(operation_desc.filter().layout));
set_argument(result, "Output", problem_space,
std::string(library::to_string(operation_desc.output().element))
std::string(library::to_string(operation_desc.output().element))
+ ":" + library::to_string(operation_desc.output().layout));
set_argument(result, "conv_kind", problem_space, library::to_string(operation_desc.conv_kind));
@@ -506,7 +506,7 @@ void Conv3dOperationProfiler::initialize_result_(
set_argument(result, "t", problem_space, problem_.t);
set_argument(result, "r", problem_space, problem_.r);
set_argument(result, "s", problem_space, problem_.s);
set_argument(result, "z", problem_space, problem_.z);
set_argument(result, "p", problem_space, problem_.p);
set_argument(result, "q", problem_space, problem_.q);
@@ -523,11 +523,11 @@ void Conv3dOperationProfiler::initialize_result_(
set_argument(result, "dilation_h", problem_space, problem_.dilation_h);
set_argument(result, "dilation_w", problem_space, problem_.dilation_w);
set_argument(result, "split_k_mode", problem_space,
set_argument(result, "split_k_mode", problem_space,
std::string(library::to_string(problem_.split_k_mode)));
set_argument(result, "split_k_slices", problem_space, problem_.split_k_slices);
set_argument(result, "conv_mode", problem_space,
set_argument(result, "conv_mode", problem_space,
std::string(library::to_string(problem_.conv_mode)));
set_argument(result, "alpha", problem_space,
@@ -536,7 +536,7 @@ void Conv3dOperationProfiler::initialize_result_(
set_argument(result, "beta", problem_space,
library::lexical_cast(problem_.beta, operation_desc.element_epilogue));
set_argument(result, "eq_gemm_provider", problem_space,
set_argument(result, "eq_gemm_provider", problem_space,
std::string(library::to_string(problem_.eq_gemm_provider)));
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
@@ -554,14 +554,14 @@ void Conv3dOperationProfiler::initialize_result_(
/// Initialize reduction problem dimensions and library::Operation
bool Conv3dOperationProfiler::initialize_reduction_configuration_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::ConvDescription const &conv_desc =
library::ConvDescription const &conv_desc =
static_cast<library::ConvDescription const &>(operation->description());
library::ConvKind const &conv_kind = conv_desc.conv_kind;
@@ -585,14 +585,14 @@ bool Conv3dOperationProfiler::initialize_reduction_configuration_(
conv_workspace_.reduction_configuration.lds = conv_workspace_.configuration.layout_c(conv_kind).stride()[tensor_c_stride_idx];
conv_workspace_.reduction_configuration.ldd = conv_workspace_.configuration.layout_c(conv_kind).stride()[tensor_c_stride_idx];
// find reduction operation
// find reduction operation
library::ReductionFunctionalKey reduction_key(
library::Provider::kCUTLASS,
conv_desc.tile_description.math_instruction.element_accumulator, // element workspace
conv_desc.tile_description.math_instruction.element_accumulator, // element workspace
conv_desc.tile_description.math_instruction.element_accumulator, // element accumulator
conv_desc.C.element, // element output
conv_desc.element_epilogue // element compute
);
);
#if 0// debug print to check which reduction instance is selected
std::cout << reduction_key << "\n";
@@ -602,7 +602,7 @@ bool Conv3dOperationProfiler::initialize_reduction_configuration_(
if(reduction_it == Singleton::get().operation_table.reduction_operations.end()) {
return false;
}
}
// initialize reduction operation required for parallel split-k conv2d operator
reduction_op_ = reduction_it->second;
@@ -614,13 +614,24 @@ bool Conv3dOperationProfiler::initialize_reduction_configuration_(
/// Initializes workspace
Status Conv3dOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
// initialize conv2d underlying operation to handle parallel reduction
library::Operation const* underlying_operation = operation;
@@ -630,15 +641,15 @@ Status Conv3dOperationProfiler::initialize_workspace(
}
}
library::ConvDescription const &operation_desc =
library::ConvDescription const &operation_desc =
static_cast<library::ConvDescription const &>(underlying_operation->description());
// Compute the number of copies of the problem to avoid L2 camping.
if (!options.profiling.workspace_count) {
int64_t bytes = problem_.bytes(operation_desc);
if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
if (bytes < 3 * int64_t(options.device.properties[0].l2CacheSize)) {
conv_workspace_.problem_count =
1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
1 + int((3 * int64_t(options.device.properties[0].l2CacheSize)) / bytes);
}
else {
conv_workspace_.problem_count = 1;
@@ -651,7 +662,7 @@ Status Conv3dOperationProfiler::initialize_workspace(
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
conv_workspace_.A = device_context.allocate_tensor(
conv_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -659,10 +670,11 @@ Status Conv3dOperationProfiler::initialize_workspace(
problem_.extent_a(operation_desc.conv_kind),
conv_workspace_.stride_a(operation_desc.conv_kind),
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
conv_workspace_.B = device_context.allocate_tensor(
conv_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -670,10 +682,11 @@ Status Conv3dOperationProfiler::initialize_workspace(
problem_.extent_b(operation_desc.conv_kind),
conv_workspace_.stride_b(operation_desc.conv_kind),
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
conv_workspace_.C = device_context.allocate_tensor(
conv_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -681,27 +694,32 @@ Status Conv3dOperationProfiler::initialize_workspace(
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.stride_c(operation_desc.conv_kind),
conv_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
conv_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.stride_c(operation_desc.conv_kind),
conv_workspace_.problem_count
conv_workspace_.problem_count,
0 // device_index
);
conv_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
problem_.extent_c(operation_desc.conv_kind),
conv_workspace_.stride_c(operation_desc.conv_kind),
conv_workspace_.problem_count
conv_workspace_.problem_count,
0 // device_index
);
}
//
@@ -733,10 +751,10 @@ Status Conv3dOperationProfiler::initialize_workspace(
conv_workspace_.reduction_host_workspace.resize(workspace_size, 0);
status = reduction_op_->initialize(
&conv_workspace_.reduction_configuration,
conv_workspace_.reduction_host_workspace.data(),
&conv_workspace_.reduction_configuration,
conv_workspace_.reduction_host_workspace.data(),
nullptr);
if (status != Status::kSuccess) {
return status;
}
@@ -763,7 +781,7 @@ Status Conv3dOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool Conv3dOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -784,7 +802,7 @@ bool Conv3dOperationProfiler::verify_cutlass(
set_cutlass_operator_arguments_();
conv_workspace_.Computed->copy_from_device(conv_workspace_.C->data());
//
// Run the CUTLASS operation
//
@@ -799,9 +817,9 @@ bool Conv3dOperationProfiler::verify_cutlass(
}
#if 0
std::cout << "profiling : " << std::endl
<< "conv2d : " << operation->description().name << std::endl
<< "underlying conv2d : " << underlying_operation->description().name << std::endl
std::cout << "profiling : " << std::endl
<< "conv2d : " << operation->description().name << std::endl
<< "underlying conv2d : " << underlying_operation->description().name << std::endl
<< "reduction : " << reduction_op_->description().name << std::endl;
#endif
@@ -818,7 +836,7 @@ bool Conv3dOperationProfiler::verify_cutlass(
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
results_.back().status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
conv_workspace_.reduction_host_workspace.data(),
@@ -840,7 +858,7 @@ bool Conv3dOperationProfiler::verify_cutlass(
// CUTLASS op ran the but not yet verified against any verification provider
results_.back().disposition = Disposition::kNotVerified;
//
// Run verification providers
//
@@ -856,7 +874,7 @@ bool Conv3dOperationProfiler::verify_cutlass(
Status status = cudnn_satisfies(conv_desc, conv_workspace_.configuration);
// Initialize reference data to the source data
// Initialize reference data to the source data
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
if (status == Status::kSuccess) {
@@ -883,8 +901,8 @@ bool Conv3dOperationProfiler::verify_cutlass(
// Run verification host reference
if (options.verification.provider_enabled(library::Provider::kReferenceHost)) {
// Restore reference data back to initial source data
// Restore reference data back to initial source data
conv_workspace_.Reference->copy_from_device(conv_workspace_.C->data());
verify_with_host_reference_(
@@ -893,10 +911,10 @@ bool Conv3dOperationProfiler::verify_cutlass(
device_context,
operation,
problem_space,
problem);
problem);
}
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -921,7 +939,7 @@ bool Conv3dOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against host reference
bool Conv3dOperationProfiler::verify_with_host_reference_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -939,14 +957,14 @@ bool Conv3dOperationProfiler::verify_with_host_reference_(
library::ConvFunctionalKey conv_key(
library::Provider::kReferenceHost,
conv_desc.conv_kind,
conv_desc.conv_kind,
conv_desc.A.element,
conv_desc.A.layout,
conv_desc.B.element,
conv_desc.B.layout,
conv_desc.C.element,
conv_desc.C.layout,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.tile_description.math_instruction.element_accumulator,
conv_desc.element_epilogue);
#if 0 // debug print to check which host reference instance is selected
@@ -959,12 +977,12 @@ bool Conv3dOperationProfiler::verify_with_host_reference_(
results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
return true;
}
}
// conv3d host reference minimum cc is 0 (CPU) and no iterator algorithm
library::ConvPreferenceKey preference_key(0, library::IteratorAlgorithmID::kNone);
auto cc_it = operators_it->second.find(preference_key);
if(cc_it == operators_it->second.end()) {
results_.back().verification_map[library::Provider::kReferenceHost] = Disposition::kNotRun;
return true;
@@ -1035,9 +1053,9 @@ bool Conv3dOperationProfiler::verify_with_host_reference_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kReferenceHost] == Disposition::kIncorrect) {
save_workspace(
device_context,
options,
@@ -1053,7 +1071,7 @@ bool Conv3dOperationProfiler::verify_with_host_reference_(
/// Verifies CUTLASS against host reference
bool Conv3dOperationProfiler::verify_with_device_reference_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -1068,14 +1086,14 @@ bool Conv3dOperationProfiler::verify_with_device_reference_(
/// Measures performance results
bool Conv3dOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
set_cutlass_operator_arguments_();
@@ -1180,7 +1198,7 @@ Status Conv3dOperationProfiler::profile_cutlass_(
return status;
}
}
//
// Initialize GPU timer
//
@@ -1198,9 +1216,9 @@ Status Conv3dOperationProfiler::profile_cutlass_(
// Setup rotating workspace
int problem_idx = (iteration % conv_workspace_.problem_count);
set_cutlass_operator_arguments_(problem_idx);
// Run underlying conv2d operation
status = underlying_operation->run(
arguments,
@@ -1208,7 +1226,7 @@ Status Conv3dOperationProfiler::profile_cutlass_(
device_workspace);
// Run parallel reduction kernel for parallel split_k_mode
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
if (conv_workspace_.configuration.split_k_mode == conv::SplitKMode::kParallel) {
status = reduction_op_->run(
&conv_workspace_.reduction_arguments,
conv_workspace_.reduction_host_workspace.data(),
@@ -1229,7 +1247,7 @@ Status Conv3dOperationProfiler::profile_cutlass_(
//
// Update performance result
//
runtime = timer.duration(iteration);
return status;
@@ -1240,7 +1258,7 @@ Status Conv3dOperationProfiler::profile_cutlass_(
/// Verifies CUTLASS against cudnn reference
bool Conv3dOperationProfiler::verify_with_cudnn_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -1257,7 +1275,7 @@ bool Conv3dOperationProfiler::verify_with_cudnn_(
cudnnStatus_t status = handle.get_cudnn_create_status();
if (status != CUDNN_STATUS_SUCCESS) {
results_.back().verification_map[library::Provider::kCUDNN] = get_cutlass_disposition(status);
return true;
}
@@ -1285,8 +1303,8 @@ bool Conv3dOperationProfiler::verify_with_cudnn_(
// Construct dispatcher to cudnn operator
//
detail::cudnnConvDispatcher conv_op(
conv_desc,
detail::cudnnConvDispatcher conv_op(
conv_desc,
conv_workspace_.configuration,
conv_workspace_.arguments,
handle
@@ -1323,7 +1341,7 @@ bool Conv3dOperationProfiler::verify_with_cudnn_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUDNN] == Disposition::kIncorrect) {
save_workspace(
tools/profiler/src/cublas_helpers.cu
+274 -1
View File
@@ -259,6 +259,25 @@ Status cublas_satisfies(library::GemmDescription const &desc) {
return Status::kErrorNotSupported;
}
// Refer to https://docs.nvidia.com/cuda/cublas/#id105
// input type A and B FE5M2 not supported in cuBLASLt
if(desc.A.element == library::NumericTypeID::kFE5M2 &&
desc.B.element == library::NumericTypeID::kFE5M2){
return Status::kErrorNotSupported;
}
// Refer to https://docs.nvidia.com/cuda/cublas/#id105
// input type A and B are FE5M2 and FE4M3 then D type should be F32
if (desc.A.element == library::NumericTypeID::kFE5M2 &&
desc.B.element == library::NumericTypeID::kFE4M3 &&
desc.C.element == library::NumericTypeID::kF32 &&
desc.D.element != library::NumericTypeID::kF32 ){
return Status::kErrorNotSupported;
}
// output type S4 and S8 not supported in cuBLAS
if (desc.C.element == library::NumericTypeID::kS4 ||
desc.C.element == library::NumericTypeID::kS8) {
@@ -405,7 +424,261 @@ cublasStatus_t cublasGemmExDispatcher::operator()(cublasHandle_t handle) {
}
}
} // namespace detail
cublasLtGemmExDispatcher::cublasLtGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmUniversalConfiguration configuration_,
library::GemmUniversalArguments arguments_
):
op_desc(op_desc), configuration(configuration_), arguments(arguments_), status(Status::kSuccess) {
bool good = true;
good = (good && get_cublas_transpose_operation(trans_A, op_desc.A.layout, op_desc.transform_A));
good = (good && get_cublas_transpose_operation(trans_B, op_desc.B.layout, op_desc.transform_B));
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
good = (good && get_cublas_datatype(
compute_data_type,
op_desc.tile_description.math_instruction.element_accumulator));
// cuBLAS introduces a separate cublasComputeType enumerant to more precisely describe
// internal numerical data types used in the computation.
#if (__CUDACC_VER_MAJOR__ >= 11)
library::OpcodeClassID const & opcode_class =
op_desc.tile_description.math_instruction.opcode_class;
if (good &&
op_desc.A.element == library::NumericTypeID::kF32 &&
op_desc.B.element == library::NumericTypeID::kF32 &&
opcode_class == library::OpcodeClassID::kTensorOp) {
compute_type = CUBLAS_COMPUTE_32F_FAST_TF32;
}
else if (good) {
bool const isPedantic = false;
switch (compute_data_type) {
case CUDA_R_32F:
case CUDA_C_32F:
compute_type = isPedantic ? CUBLAS_COMPUTE_32F_PEDANTIC : CUBLAS_COMPUTE_32F;
break;
case CUDA_R_64F:
case CUDA_C_64F:
compute_type = isPedantic ? CUBLAS_COMPUTE_64F_PEDANTIC : CUBLAS_COMPUTE_64F;
break;
case CUDA_R_16F:
compute_type = isPedantic ? CUBLAS_COMPUTE_16F_PEDANTIC : CUBLAS_COMPUTE_16F;
break;
case CUDA_R_32I:
compute_type = isPedantic ? CUBLAS_COMPUTE_32I_PEDANTIC : CUBLAS_COMPUTE_32I;
break;
default:
good = false;
break;
}
}
#endif // __CUDACC_VER_MAJOR__ >= 11
if (!good) {
status = Status::kErrorNotSupported;
}
}
void cublasLtGemmExDispatcher::initialize_cublaslt(){
// create operation desciriptor; see cublasLtMatmulDescAttributes_t for details about defaults; here we just need to
// set the transforms for A and B
cublasLtMatmulDescCreate(&operationDesc, compute_type, compute_data_type);
cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_A, sizeof(trans_A));
cublasLtMatmulDescSetAttribute(operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &trans_B, sizeof(trans_B));
uint64_t contiguous_A = (trans_A == CUBLAS_OP_N ? configuration.problem_size.m() : configuration.problem_size.k());
uint64_t strided_A = (trans_A == CUBLAS_OP_N ? configuration.problem_size.k() : configuration.problem_size.m());
uint64_t contiguous_B = (trans_B == CUBLAS_OP_N ? configuration.problem_size.k() : configuration.problem_size.n());
uint64_t strided_B = (trans_B == CUBLAS_OP_N ? configuration.problem_size.n() : configuration.problem_size.k());
// create matrix descriptors, we are good with the details here so no need to set any extra attributes
// table of supported type combinations can be found in the documentation: https://docs.nvidia.com/cuda/cublas/index.html#cublasltmatmul
cublasLtMatrixLayoutCreate(&Adesc, data_type_A, contiguous_A, strided_A, configuration.lda);
cublasLtMatrixLayoutCreate(&Bdesc, data_type_B, contiguous_B, strided_B, configuration.ldb);
cublasLtMatrixLayoutCreate(&Cdesc, data_type_C, configuration.problem_size.m(), configuration.problem_size.n(), configuration.ldc);
cublasLtMatrixLayoutCreate(&Ddesc, data_type_C, configuration.problem_size.m(), configuration.problem_size.n(), configuration.ldd);
}
bool cublasLtGemmExDispatcher::get_cublaslt_algo(cublasLtHandle_t handle,
AlgorithmMode algorithm_mode
){
const int requestedAlgoCount = 8; //By default gets 8 algorithms from GetHeuristic Call. CublasLt heuristics provide at max 8 algorithms.
int returnedResults = 0;
cublasLtMatmulHeuristicResult_t heuristicResult[requestedAlgoCount] = {};
#if (__CUDACC_VER_MAJOR__ >= 12)
//Decide based upon the unique operation identifier whether to turn on fast accum for cublas kernel or not.
std::string operation_name(op_desc.name);
if(operation_name.find("fastaccum") != std::string::npos){
const int8_t fastAccuMode = 1;
cublasLtMatmulDescSetAttribute(operationDesc,
CUBLASLT_MATMUL_DESC_FAST_ACCUM,
&fastAccuMode,
sizeof(fastAccuMode));
}
#endif // __CUDACC_VER_MAJOR__ >= 12
//Using 32MB for hopper kernel. This is the max workspace size for the call to cublasLtMatmulAlgoGetHeuristic()
size_t workspaceSizeForHeuristics = 32ULL * 1024 * 1024;
void* workspaceHeuristic = nullptr;
cudaError_t result = cudaMalloc((void **)&workspaceHeuristic, workspaceSizeForHeuristics);
if (result != cudaSuccess) {
throw std::bad_alloc();
}
// create preference handle; here we could use extra attributes to disable tensor ops or to make sure algo selected
// will work with badly aligned A, B, C; here for simplicity we just assume A,B,C are always well aligned (e.g.
// directly come from cudaMalloc)
cublasLtMatmulPreferenceCreate(&preference);
cublasLtMatmulPreferenceSetAttribute(preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSizeForHeuristics, sizeof(workspaceSizeForHeuristics));
cublasLtMatmulAlgoGetHeuristic(handle, operationDesc, Adesc, Bdesc, Cdesc, Ddesc, preference, requestedAlgoCount, heuristicResult, &returnedResults);
if (returnedResults == 0) {
return false;
}
int bestAlgoIdx = 0;
//
//Auto Tuning to get the best kernel for the given problem
//
if (algorithm_mode == AlgorithmMode::kBest) {
float time = 0;
float bestAlgoTime = 0;
cudaStream_t stream;
cudaEvent_t startEvent, stopEvent;
cudaStreamCreate(&stream);
cudaEventCreate(&startEvent);
cudaEventCreate(&stopEvent);
constexpr int repeatAlgoCheck = 5;
std::vector<float> algoTimes(repeatAlgoCheck);
for (int algoIdx = 0; algoIdx < returnedResults; algoIdx++) {
for (int checkIdx = 0; checkIdx < repeatAlgoCheck; checkIdx++) {
cudaEventRecord(startEvent, stream);
cublasStatus_t status = cublasLtMatmul(handle,
operationDesc,
arguments.alpha,
arguments.A,
Adesc,
arguments.B,
Bdesc,
arguments.beta,
arguments.C,
Cdesc,
arguments.D,
Ddesc,
&heuristicResult[algoIdx].algo,
workspaceHeuristic,
heuristicResult[algoIdx].workspaceSize,
stream);
// Handle errors
if (status != CUBLAS_STATUS_SUCCESS) {
std::cerr << "cublasLtMatmul AutoTuning failed with status: " << cublasLtGetStatusName(status) << std::endl;
return false;
}
cudaEventRecord(stopEvent, stream);
cudaEventSynchronize(stopEvent);
cudaEventElapsedTime(&time, startEvent, stopEvent);
algoTimes[checkIdx] = time;
}
const size_t size = algoTimes.size();
if (size == 0) {
time = 0;
}
std::sort(algoTimes.begin(), algoTimes.end());
const size_t mid = size / 2;
if (size % 2 == 0) {
time = (algoTimes[mid] + algoTimes[mid - 1]) / 2;
}
else {
time = algoTimes[mid];
}
if (algoIdx == 0 || time < bestAlgoTime) {
bestAlgoTime = time;
bestAlgoIdx = algoIdx;
}
}
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cout << "\n";
std::cout << "# Algorithms checked: " << returnedResults << "\n";
std::cout << "WorkspaceSize Allocated: " << heuristicResult[bestAlgoIdx].workspaceSize << "\n";
std::cout << "Algorithm selected after auto-tuning is:" << "\n";
int algoId, tile, swizzle, customOption, numSplitsK, reductionScheme;
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_ID, &algoId, sizeof(algoId), NULL);
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_TILE_ID, &tile, sizeof(tile), NULL);
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_SPLITK_NUM, &numSplitsK, sizeof(numSplitsK), NULL);
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_REDUCTION_SCHEME, &reductionScheme, sizeof(reductionScheme), NULL);
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_CTA_SWIZZLING, &swizzle, sizeof(swizzle), NULL);
cublasLtMatmulAlgoConfigGetAttribute(&heuristicResult[bestAlgoIdx].algo, CUBLASLT_ALGO_CONFIG_CUSTOM_OPTION, &customOption, sizeof(customOption), NULL);
printf("algo={ Id=%d, tileIdx=%d splitK=%d reduc=%d swizzle=%d custom=%d }\n",
algoId, tile, numSplitsK, reductionScheme, swizzle, customOption);
#endif
if (stream) cudaStreamDestroy(stream);
if (startEvent) cudaEventDestroy(startEvent);
if (stopEvent) cudaEventDestroy(stopEvent);
}
//setting algorithm for the dispatcher
heuristicResult_ = heuristicResult[bestAlgoIdx];
result = cudaMalloc((void **)&workspace, heuristicResult_.workspaceSize);
if (result != cudaSuccess) {
throw std::bad_alloc();
}
return true;
}
cublasStatus_t cublasLtGemmExDispatcher::operator()(cublasLtHandle_t handle)
{
return cublasLtMatmul(handle,
operationDesc,
arguments.alpha,
arguments.A,
Adesc,
arguments.B,
Bdesc,
arguments.beta,
arguments.C,
Cdesc,
arguments.D,
Ddesc,
&heuristicResult_.algo,
workspace,
heuristicResult_.workspaceSize,
0); //number of streams is set to 0
}
}
// namespace detail
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/cutlass_profiler.cu
-13
View File
@@ -208,19 +208,6 @@ void CutlassProfiler::print_options_(std::ostream &out) {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Initializes the CUDA device
void CutlassProfiler::initialize_device_() {
cudaError_t result = cudaSetDevice(options_.device.device);
if (result != cudaSuccess) {
std::cerr << "Failed to set device.";
throw std::runtime_error("Failed to set device");
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
tools/profiler/src/device_allocation.cu
+226 -182
View File
@@ -88,16 +88,16 @@ static std::vector<int64_t> get_packed_layout_stride(std::vector<int> const &ext
/// Returns the stride of a packed layout
std::vector<int64_t> DeviceAllocation::get_packed_layout(
library::LayoutTypeID layout_id,
library::LayoutTypeID layout_id,
std::vector<int> const &extent) {
std::vector<int64_t> stride;
switch (layout_id) {
case library::LayoutTypeID::kColumnMajor:
case library::LayoutTypeID::kColumnMajor:
stride = get_packed_layout_stride<cutlass::layout::ColumnMajor>(extent);
break;
case library::LayoutTypeID::kRowMajor:
case library::LayoutTypeID::kRowMajor:
stride = get_packed_layout_stride<cutlass::layout::RowMajor>(extent);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK2:
@@ -159,7 +159,7 @@ std::vector<int64_t> DeviceAllocation::get_packed_layout(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Template to use CUTLASS Layout functions to
/// Template to use CUTLASS Layout functions to
template <typename Layout>
static size_t construct_layout_(
void *bytes,
@@ -177,8 +177,8 @@ static size_t construct_layout_(
stride = get_packed_layout_stride<Layout>(extent);
return construct_layout_<Layout>(
bytes,
layout_id,
bytes,
layout_id,
extent,
stride);
}
@@ -202,7 +202,7 @@ static size_t construct_layout_(
// Pack it into bytes
if (bytes) {
*reinterpret_cast<Layout *>(bytes) = layout;
*reinterpret_cast<Layout *>(bytes) = layout;
}
// Return capacity
@@ -219,10 +219,10 @@ size_t DeviceAllocation::construct_layout(
std::vector<int64_t> &stride) {
switch (layout_id) {
case library::LayoutTypeID::kColumnMajor:
case library::LayoutTypeID::kColumnMajor:
return construct_layout_<cutlass::layout::ColumnMajor>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kRowMajor:
case library::LayoutTypeID::kRowMajor:
return construct_layout_<cutlass::layout::RowMajor>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kColumnMajorInterleavedK2:
@@ -284,24 +284,26 @@ size_t DeviceAllocation::construct_layout(
/////////////////////////////////////////////////////////////////////////////////////////////////
DeviceAllocation::DeviceAllocation():
type_(library::NumericTypeID::kInvalid),
DeviceAllocation::DeviceAllocation():
type_(library::NumericTypeID::kInvalid),
batch_stride_(0),
capacity_(0),
capacity_(0),
pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown),
batch_count_(1) {
batch_count_(1),
device_(-1) {
}
DeviceAllocation::DeviceAllocation(
library::NumericTypeID type,
size_t capacity
library::NumericTypeID type,
size_t capacity,
int device
):
type_(type), batch_stride_(capacity), capacity_(capacity), pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown), batch_count_(1) {
type_(type), batch_stride_(capacity), capacity_(capacity), pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown), batch_count_(1), device_(device) {
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity));
cudaError_t result = this->malloc((void **)&pointer_, bytes(type, capacity));
if (result != cudaSuccess) {
type_ = library::NumericTypeID::kInvalid;
@@ -312,13 +314,15 @@ DeviceAllocation::DeviceAllocation(
}
DeviceAllocation::DeviceAllocation(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count
int batch_count,
int device
):
type_(type), batch_stride_(size_t(0)), capacity_(size_t(0)), pointer_(nullptr), batch_count_(1) {
type_(type), batch_stride_(size_t(0)), capacity_(size_t(0)),
pointer_(nullptr), batch_count_(1), device_(device) {
reset(type, layout_id, extent, stride, batch_count);
}
@@ -355,7 +359,7 @@ DeviceAllocation &DeviceAllocation::reset(library::NumericTypeID type, size_t ca
batch_stride_ = capacity;
capacity_ = capacity;
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type_, capacity_));
cudaError_t result = this->malloc((void **)&pointer_, bytes(type_, capacity_));
if (result != cudaSuccess) {
throw std::bad_alloc();
}
@@ -373,9 +377,9 @@ DeviceAllocation &DeviceAllocation::reset(library::NumericTypeID type, size_t ca
/// Allocates memory for a given layout and tensor
DeviceAllocation &DeviceAllocation::reset(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count) {
@@ -391,14 +395,14 @@ DeviceAllocation &DeviceAllocation::reset(
batch_count_ = batch_count;
batch_stride_ = construct_layout(
tensor_ref_buffer_.data() + sizeof(pointer_),
layout_id,
extent,
tensor_ref_buffer_.data() + sizeof(pointer_),
layout_id,
extent,
stride_);
capacity_ = batch_stride_ * batch_count_;
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity_));
cudaError_t result = this->malloc((void **)&pointer_, bytes(type, capacity_));
if (result != cudaSuccess) {
throw std::bad_alloc();
}
@@ -421,7 +425,7 @@ void *DeviceAllocation::data() const {
}
void *DeviceAllocation::batch_data(int batch_idx) const {
return static_cast<char *>(data()) + batch_stride_bytes() * batch_idx;
return static_cast<char *>(data()) + batch_stride_bytes() * batch_idx;
}
library::LayoutTypeID DeviceAllocation::layout() const {
@@ -1476,159 +1480,159 @@ void DeviceAllocation::initialize_random_sparsemeta_host(int seed, int MetaSizeI
/// Returns true if two blocks have exactly the same value
bool DeviceAllocation::block_compare_equal(
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
size_t capacity) {
switch (numeric_type) {
case library::NumericTypeID::kFE4M3:
return reference::device::BlockCompareEqual<float_e4m3_t>(
reinterpret_cast<float_e4m3_t const *>(ptr_A),
reinterpret_cast<float_e4m3_t const *>(ptr_B),
reinterpret_cast<float_e4m3_t const *>(ptr_A),
reinterpret_cast<float_e4m3_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE5M2:
return reference::device::BlockCompareEqual<float_e5m2_t>(
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF16:
return reference::device::BlockCompareEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
reinterpret_cast<half_t const *>(ptr_B),
reinterpret_cast<half_t const *>(ptr_A),
reinterpret_cast<half_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kBF16:
return reference::device::BlockCompareEqual<bfloat16_t>(
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_B),
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kTF32:
return reference::device::BlockCompareEqual<tfloat32_t>(
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_B),
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF32:
return reference::device::BlockCompareEqual<float>(
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_B),
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF32:
return reference::device::BlockCompareEqual<cutlass::complex<float> >(
reinterpret_cast<complex<float> const *>(ptr_A),
reinterpret_cast<complex<float> const *>(ptr_B),
reinterpret_cast<complex<float> const *>(ptr_A),
reinterpret_cast<complex<float> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF16:
return reference::device::BlockCompareEqual<complex<half_t>>(
reinterpret_cast<complex<half_t> const *>(ptr_A),
reinterpret_cast<complex<half_t> const *>(ptr_B),
reinterpret_cast<complex<half_t> const *>(ptr_A),
reinterpret_cast<complex<half_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCBF16:
return reference::device::BlockCompareEqual<complex<bfloat16_t>>(
reinterpret_cast<complex<bfloat16_t> const *>(ptr_A),
reinterpret_cast<complex<bfloat16_t> const *>(ptr_B),
reinterpret_cast<complex<bfloat16_t> const *>(ptr_A),
reinterpret_cast<complex<bfloat16_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCTF32:
return reference::device::BlockCompareEqual<complex<tfloat32_t>>(
reinterpret_cast<complex<tfloat32_t> const *>(ptr_A),
reinterpret_cast<complex<tfloat32_t> const *>(ptr_B),
reinterpret_cast<complex<tfloat32_t> const *>(ptr_A),
reinterpret_cast<complex<tfloat32_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kF64:
return reference::device::BlockCompareEqual<double>(
reinterpret_cast<double const *>(ptr_A),
reinterpret_cast<double const *>(ptr_B),
reinterpret_cast<double const *>(ptr_A),
reinterpret_cast<double const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF64:
return reference::device::BlockCompareEqual<complex<double>>(
reinterpret_cast<complex<double> const *>(ptr_A),
reinterpret_cast<complex<double> const *>(ptr_B),
reinterpret_cast<complex<double> const *>(ptr_A),
reinterpret_cast<complex<double> const *>(ptr_B),
capacity);
case library::NumericTypeID::kS2:
return reference::device::BlockCompareEqual<int2b_t>(
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_B),
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS4:
return reference::device::BlockCompareEqual<int4b_t>(
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_B),
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS8:
return reference::device::BlockCompareEqual<int8_t>(
reinterpret_cast<int8_t const *>(ptr_A),
reinterpret_cast<int8_t const *>(ptr_B),
reinterpret_cast<int8_t const *>(ptr_A),
reinterpret_cast<int8_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS16:
return reference::device::BlockCompareEqual<int16_t>(
reinterpret_cast<int16_t const *>(ptr_A),
reinterpret_cast<int16_t const *>(ptr_B),
reinterpret_cast<int16_t const *>(ptr_A),
reinterpret_cast<int16_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS32:
return reference::device::BlockCompareEqual<int32_t>(
reinterpret_cast<int32_t const *>(ptr_A),
reinterpret_cast<int32_t const *>(ptr_B),
reinterpret_cast<int32_t const *>(ptr_A),
reinterpret_cast<int32_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS64:
return reference::device::BlockCompareEqual<int64_t>(
reinterpret_cast<int64_t const *>(ptr_A),
reinterpret_cast<int64_t const *>(ptr_B),
reinterpret_cast<int64_t const *>(ptr_A),
reinterpret_cast<int64_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kB1:
return reference::device::BlockCompareEqual<uint1b_t>(
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_B),
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU2:
return reference::device::BlockCompareEqual<uint2b_t>(
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_B),
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU4:
return reference::device::BlockCompareEqual<uint4b_t>(
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_B),
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU8:
return reference::device::BlockCompareEqual<uint8_t>(
reinterpret_cast<uint8_t const *>(ptr_A),
reinterpret_cast<uint8_t const *>(ptr_B),
reinterpret_cast<uint8_t const *>(ptr_A),
reinterpret_cast<uint8_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU16:
return reference::device::BlockCompareEqual<uint16_t>(
reinterpret_cast<uint16_t const *>(ptr_A),
reinterpret_cast<uint16_t const *>(ptr_B),
reinterpret_cast<uint16_t const *>(ptr_A),
reinterpret_cast<uint16_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU32:
return reference::device::BlockCompareEqual<uint32_t>(
reinterpret_cast<uint32_t const *>(ptr_A),
reinterpret_cast<uint32_t const *>(ptr_B),
reinterpret_cast<uint32_t const *>(ptr_A),
reinterpret_cast<uint32_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU64:
return reference::device::BlockCompareEqual<uint64_t>(
reinterpret_cast<uint64_t const *>(ptr_A),
reinterpret_cast<uint64_t const *>(ptr_B),
reinterpret_cast<uint64_t const *>(ptr_A),
reinterpret_cast<uint64_t const *>(ptr_B),
capacity);
default:
@@ -1638,9 +1642,9 @@ bool DeviceAllocation::block_compare_equal(
/// Returns true if two blocks have approximately the same value
bool DeviceAllocation::block_compare_relatively_equal(
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
library::NumericTypeID numeric_type,
void const *ptr_A,
void const *ptr_B,
size_t capacity,
double epsilon,
double nonzero_floor) {
@@ -1648,161 +1652,161 @@ bool DeviceAllocation::block_compare_relatively_equal(
switch (numeric_type) {
case library::NumericTypeID::kFE4M3:
return reference::device::BlockCompareRelativelyEqual<float_e4m3_t>(
reinterpret_cast<float_e4m3_t const *>(ptr_A),
reinterpret_cast<float_e4m3_t const *>(ptr_A),
reinterpret_cast<float_e4m3_t const *>(ptr_B),
capacity,
static_cast<float_e4m3_t>(epsilon),
capacity,
static_cast<float_e4m3_t>(epsilon),
static_cast<float_e4m3_t>(nonzero_floor));
case library::NumericTypeID::kFE5M2:
return reference::device::BlockCompareRelativelyEqual<float_e5m2_t>(
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
capacity,
static_cast<float_e5m2_t>(epsilon),
capacity,
static_cast<float_e5m2_t>(epsilon),
static_cast<float_e5m2_t>(nonzero_floor));
case library::NumericTypeID::kF16:
return reference::device::BlockCompareRelativelyEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
reinterpret_cast<half_t const *>(ptr_A),
reinterpret_cast<half_t const *>(ptr_B),
capacity,
static_cast<half_t>(epsilon),
capacity,
static_cast<half_t>(epsilon),
static_cast<half_t>(nonzero_floor));
case library::NumericTypeID::kBF16:
return reference::device::BlockCompareRelativelyEqual<bfloat16_t>(
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_B),
capacity,
static_cast<bfloat16_t>(epsilon),
capacity,
static_cast<bfloat16_t>(epsilon),
static_cast<bfloat16_t>(nonzero_floor));
case library::NumericTypeID::kTF32:
return reference::device::BlockCompareRelativelyEqual<tfloat32_t>(
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_B),
capacity,
static_cast<tfloat32_t>(epsilon),
capacity,
static_cast<tfloat32_t>(epsilon),
static_cast<tfloat32_t>(nonzero_floor));
case library::NumericTypeID::kF32:
return reference::device::BlockCompareRelativelyEqual<float>(
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_B),
capacity,
static_cast<float>(epsilon),
capacity,
static_cast<float>(epsilon),
static_cast<float>(nonzero_floor));
case library::NumericTypeID::kF64:
return reference::device::BlockCompareRelativelyEqual<double>(
reinterpret_cast<double const *>(ptr_A),
reinterpret_cast<double const *>(ptr_A),
reinterpret_cast<double const *>(ptr_B),
capacity,
static_cast<double>(epsilon),
capacity,
static_cast<double>(epsilon),
static_cast<double>(nonzero_floor));
case library::NumericTypeID::kS2:
return reference::device::BlockCompareRelativelyEqual<int2b_t>(
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_B),
capacity,
static_cast<int2b_t>(epsilon),
capacity,
static_cast<int2b_t>(epsilon),
static_cast<int2b_t>(nonzero_floor));
case library::NumericTypeID::kS4:
return reference::device::BlockCompareRelativelyEqual<int4b_t>(
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_B),
capacity,
static_cast<int4b_t>(epsilon),
capacity,
static_cast<int4b_t>(epsilon),
static_cast<int4b_t>(nonzero_floor));
case library::NumericTypeID::kS8:
return reference::device::BlockCompareRelativelyEqual<int8_t>(
reinterpret_cast<int8_t const *>(ptr_A),
reinterpret_cast<int8_t const *>(ptr_A),
reinterpret_cast<int8_t const *>(ptr_B),
capacity,
static_cast<int8_t>(epsilon),
capacity,
static_cast<int8_t>(epsilon),
static_cast<int8_t>(nonzero_floor));
case library::NumericTypeID::kS16:
return reference::device::BlockCompareRelativelyEqual<int16_t>(
reinterpret_cast<int16_t const *>(ptr_A),
reinterpret_cast<int16_t const *>(ptr_A),
reinterpret_cast<int16_t const *>(ptr_B),
capacity,
static_cast<int16_t>(epsilon),
capacity,
static_cast<int16_t>(epsilon),
static_cast<int16_t>(nonzero_floor));
case library::NumericTypeID::kS32:
return reference::device::BlockCompareRelativelyEqual<int32_t>(
reinterpret_cast<int32_t const *>(ptr_A),
reinterpret_cast<int32_t const *>(ptr_A),
reinterpret_cast<int32_t const *>(ptr_B),
capacity,
static_cast<int32_t>(epsilon),
capacity,
static_cast<int32_t>(epsilon),
static_cast<int32_t>(nonzero_floor));
case library::NumericTypeID::kS64:
return reference::device::BlockCompareRelativelyEqual<int64_t>(
reinterpret_cast<int64_t const *>(ptr_A),
reinterpret_cast<int64_t const *>(ptr_A),
reinterpret_cast<int64_t const *>(ptr_B),
capacity,
static_cast<int64_t>(epsilon),
capacity,
static_cast<int64_t>(epsilon),
static_cast<int64_t>(nonzero_floor));
case library::NumericTypeID::kB1:
return reference::device::BlockCompareRelativelyEqual<uint1b_t>(
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_B),
capacity,
static_cast<uint1b_t>(epsilon),
capacity,
static_cast<uint1b_t>(epsilon),
static_cast<uint1b_t>(nonzero_floor));
case library::NumericTypeID::kU2:
return reference::device::BlockCompareRelativelyEqual<uint2b_t>(
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_B),
capacity,
static_cast<uint2b_t>(epsilon),
capacity,
static_cast<uint2b_t>(epsilon),
static_cast<uint2b_t>(nonzero_floor));
case library::NumericTypeID::kU4:
return reference::device::BlockCompareRelativelyEqual<uint4b_t>(
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_B),
capacity,
static_cast<uint4b_t>(epsilon),
capacity,
static_cast<uint4b_t>(epsilon),
static_cast<uint4b_t>(nonzero_floor));
case library::NumericTypeID::kU8:
return reference::device::BlockCompareRelativelyEqual<uint8_t>(
reinterpret_cast<uint8_t const *>(ptr_A),
reinterpret_cast<uint8_t const *>(ptr_A),
reinterpret_cast<uint8_t const *>(ptr_B),
capacity,
static_cast<uint8_t>(epsilon),
capacity,
static_cast<uint8_t>(epsilon),
static_cast<uint8_t>(nonzero_floor));
case library::NumericTypeID::kU16:
return reference::device::BlockCompareRelativelyEqual<uint16_t>(
reinterpret_cast<uint16_t const *>(ptr_A),
reinterpret_cast<uint16_t const *>(ptr_A),
reinterpret_cast<uint16_t const *>(ptr_B),
capacity,
static_cast<uint16_t>(epsilon),
capacity,
static_cast<uint16_t>(epsilon),
static_cast<uint16_t>(nonzero_floor));
case library::NumericTypeID::kU32:
return reference::device::BlockCompareRelativelyEqual<uint32_t>(
reinterpret_cast<uint32_t const *>(ptr_A),
reinterpret_cast<uint32_t const *>(ptr_A),
reinterpret_cast<uint32_t const *>(ptr_B),
capacity,
static_cast<uint32_t>(epsilon),
capacity,
static_cast<uint32_t>(epsilon),
static_cast<uint32_t>(nonzero_floor));
case library::NumericTypeID::kU64:
return reference::device::BlockCompareRelativelyEqual<uint64_t>(
reinterpret_cast<uint64_t const *>(ptr_A),
reinterpret_cast<uint64_t const *>(ptr_A),
reinterpret_cast<uint64_t const *>(ptr_B),
capacity,
static_cast<uint64_t>(epsilon),
capacity,
static_cast<uint64_t>(epsilon),
static_cast<uint64_t>(nonzero_floor));
// No relatively equal comparison for complex numbers.
@@ -1821,7 +1825,7 @@ bool DeviceAllocation::block_compare_relatively_equal(
reinterpret_cast<complex<float> const *>(ptr_A),
reinterpret_cast<complex<float> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF64:
return reference::device::BlockCompareEqual<cutlass::complex<double> >(
reinterpret_cast<complex<double> const *>(ptr_A),
@@ -1837,14 +1841,14 @@ bool DeviceAllocation::block_compare_relatively_equal(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Permits copying dynamic vectors into static-length vectors
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord, int Rank>
struct vector_to_coord {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
coord[Rank - 1] = vec.at(Rank - 1);
if (Rank > 1) {
vector_to_coord<TensorCoord, Rank - 1>(coord, vec);
}
@@ -1853,17 +1857,17 @@ struct vector_to_coord {
vector_to_coord(TensorCoord &coord, std::vector<int64_t> const &vec) {
coord[Rank - 1] = (int)vec.at(Rank - 1);
if (Rank > 1) {
vector_to_coord<TensorCoord, Rank - 1>(coord, vec);
}
}
};
/// Permits copying dynamic vectors into static-length vectors
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord>
struct vector_to_coord<TensorCoord, 1> {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
coord[0] = vec.at(0);
@@ -1875,10 +1879,10 @@ struct vector_to_coord<TensorCoord, 1> {
}
};
/// Permits copying dynamic vectors into static-length vectors
/// Permits copying dynamic vectors into static-length vectors
template <typename TensorCoord>
struct vector_to_coord<TensorCoord, 0> {
vector_to_coord(TensorCoord &coord, std::vector<int> const &vec) {
}
@@ -1888,7 +1892,7 @@ struct vector_to_coord<TensorCoord, 0> {
template <typename Element, typename Layout>
static void write_tensor_csv_static_tensor_view(
std::ostream &out,
std::ostream &out,
DeviceAllocation &allocation) {
Coord<Layout::kRank> extent;
@@ -1903,7 +1907,7 @@ static void write_tensor_csv_static_tensor_view(
}
vector_to_coord<Coord<Layout::kRank>, Layout::kRank>(extent, allocation.extent());
vector_to_coord<Coord<Layout::kStrideRank, typename Layout::Stride::Index>,
vector_to_coord<Coord<Layout::kStrideRank, typename Layout::Stride::Index>,
Layout::kStrideRank>(stride, allocation.stride());
Layout layout(stride);
@@ -1914,7 +1918,7 @@ static void write_tensor_csv_static_tensor_view(
}
host_tensor.copy_in_device_to_host(
static_cast<Element const *>(allocation.data()),
static_cast<Element const *>(allocation.data()),
allocation.batch_stride());
TensorViewWrite(out, host_tensor.host_view());
@@ -1926,7 +1930,7 @@ static void write_tensor_csv_static_tensor_view(
template <typename T>
static void write_tensor_csv_static_type(
std::ostream &out,
std::ostream &out,
DeviceAllocation &allocation) {
switch (allocation.layout()) {
@@ -1991,7 +1995,7 @@ static void write_tensor_csv_static_type(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Writes a tensor to csv
/// Writes a tensor to csv
void DeviceAllocation::write_tensor_csv(
std::ostream &out) {
@@ -1999,14 +2003,14 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kFE4M3:
write_tensor_csv_static_type<float_e4m3_t>(out, *this);
break;
case library::NumericTypeID::kFE5M2:
write_tensor_csv_static_type<float_e5m2_t>(out, *this);
break;
case library::NumericTypeID::kF16:
write_tensor_csv_static_type<half_t>(out, *this);
break;
case library::NumericTypeID::kBF16:
write_tensor_csv_static_type<bfloat16_t>(out, *this);
break;
@@ -2022,7 +2026,7 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kF64:
write_tensor_csv_static_type<double>(out, *this);
break;
case library::NumericTypeID::kS2:
write_tensor_csv_static_type<int2b_t>(out, *this);
break;
@@ -2046,7 +2050,7 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kS64:
write_tensor_csv_static_type<int64_t>(out, *this);
break;
case library::NumericTypeID::kB1:
write_tensor_csv_static_type<uint1b_t>(out, *this);
break;
@@ -2074,7 +2078,7 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kU64:
write_tensor_csv_static_type<uint64_t>(out, *this);
break;
case library::NumericTypeID::kCF16:
write_tensor_csv_static_type<cutlass::complex<half_t> >(out, *this);
break;
@@ -2110,7 +2114,7 @@ static void tensor_fill_tensor_view(DeviceAllocation &allocation, Element val =
}
vector_to_coord<Coord<Layout::kRank>, Layout::kRank>(extent, allocation.extent());
vector_to_coord<Coord<Layout::kStrideRank, typename Layout::LongIndex>,
vector_to_coord<Coord<Layout::kStrideRank, typename Layout::LongIndex>,
Layout::kStrideRank>(stride, allocation.stride());
TensorView<Element, Layout> view(
@@ -2432,6 +2436,46 @@ void DeviceAllocation::fill_host(double val = 0.0) {
copy_from_host(host_data.data());
}
cudaError_t DeviceAllocation::malloc(void** ptr, size_t size) {
cudaError_t result;
int set_device_back_to = -1;
/// When needed this sets the device to the allocation's device remembering
/// the current device so that it can be set back after the cudaMalloc is
/// performed.
if (device_ >= 0) {
int current_device;
result = cudaGetDevice(&current_device);
if (result != cudaSuccess) {
return result;
}
if (current_device != device_) {
set_device_back_to = current_device;
result = cudaSetDevice(device_);
if (result != cudaSuccess) {
return result;
}
}
}
// This performs the cudaMalloc
result = cudaMalloc(ptr, size);
if (result != cudaSuccess) {
return result;
}
/// When needed this sets the device back to what it was when the function was
/// called.
if (set_device_back_to != -1) {
result = cudaSetDevice(set_device_back_to);
if (result != cudaSuccess) {
return result;
}
}
return cudaSuccess;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/device_context.cu
+53 -42
View File
@@ -29,7 +29,7 @@
*
**************************************************************************************************/
/* \file
\brief
\brief
*/
#include "cutlass/profiler/device_context.h"
@@ -41,29 +41,16 @@ namespace profiler {
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *DeviceContext::allocate_block(
Options const &options,
std::string const &name,
library::NumericTypeID type,
size_t capacity) {
library::NumericTypeID type,
size_t capacity,
size_t device_index) {
device_memory_.emplace_back(type, capacity);
int device = options.device.device_id(device_index);
device_memory_.emplace_back(type, capacity, device);
DeviceAllocation *allocation = &device_memory_.back();
allocations_[name] = allocation;
return allocation;
}
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *DeviceContext::allocate_tensor(
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count) {
device_memory_.emplace_back(type, layout_id, extent, stride, batch_count);
DeviceAllocation *allocation = &device_memory_.back();
allocations_[name] = allocation;
return allocation;
}
@@ -72,18 +59,40 @@ DeviceAllocation *DeviceContext::allocate_tensor(
DeviceAllocation *DeviceContext::allocate_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count,
int seed_shift) {
size_t device_index) {
DeviceAllocation *allocation =
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
int device = options.device.device_id(device_index);
device_memory_.emplace_back(type, layout_id, extent, stride, batch_count,
device);
DeviceAllocation *allocation = &device_memory_.back();
allocations_[name] = allocation;
return allocation;
}
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *DeviceContext::allocate_and_initialize_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count,
int seed_shift,
size_t device_index) {
DeviceAllocation *allocation =
allocate_tensor(options, name, type, layout_id, extent, stride,
batch_count, device_index);
if (options.initialization.enabled) {
Distribution data_distribution = options.initialization.data_distribution;
Distribution data_distribution = options.initialization.data_distribution;
// check if data distribution is allowed to change
if(!options.initialization.fix_data_distribution) {
@@ -129,13 +138,13 @@ DeviceAllocation *DeviceContext::allocate_tensor(
double stddev = data_distribution.gaussian.stddev;
int scale = data_distribution.int_scale;
if (name == "A" && data_distribution.gaussian.pnzA != 100.0) {
if (name == "A" && data_distribution.gaussian.pnzA != 1.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzA);
}
else if (name == "B" && data_distribution.gaussian.pnzB != 100.0) {
else if (name == "B" && data_distribution.gaussian.pnzB != 1.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzB);
}
else if (name == "C" && data_distribution.gaussian.pnzC != 100.0) {
else if (name == "C" && data_distribution.gaussian.pnzC != 1.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzC);
}
}
@@ -147,7 +156,7 @@ DeviceAllocation *DeviceContext::allocate_tensor(
}
else {
allocation->initialize_random_device(
options.initialization.seed + seed_shift,
options.initialization.seed + seed_shift,
data_distribution);
}
}
@@ -158,7 +167,7 @@ DeviceAllocation *DeviceContext::allocate_tensor(
}
else {
allocation->initialize_random_host(
options.initialization.seed + seed_shift,
options.initialization.seed + seed_shift,
data_distribution);
}
}
@@ -167,20 +176,22 @@ DeviceAllocation *DeviceContext::allocate_tensor(
return allocation;
}
/// Allocates memory for sparse meta data
DeviceAllocation *DeviceContext::allocate_sparsemeta_tensor(
/// Allocates memory for sparse meta data
DeviceAllocation *DeviceContext::allocate_and_initialize_sparsemeta_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
library::NumericTypeID type_a,
std::vector<int> const &extent,
std::vector<int> const &extent,
std::vector<int64_t> const &stride,
int batch_count,
int seed_shift) {
int seed_shift,
size_t device_index) {
DeviceAllocation *allocation =
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
DeviceAllocation *allocation =
allocate_tensor(options, name, type, layout_id, extent, stride,
batch_count, device_index);
if (options.initialization.enabled) {
// TF32 has 4bit meta data. The rest has 2bit.
@@ -188,12 +199,12 @@ DeviceAllocation *DeviceContext::allocate_sparsemeta_tensor(
if (options.initialization.provider == library::Provider::kReferenceDevice) {
allocation->initialize_random_sparsemeta_device(
options.initialization.seed + seed_shift,
options.initialization.seed + seed_shift,
MetaSizeInBits);
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
allocation->initialize_random_sparsemeta_host(
options.initialization.seed + seed_shift,
options.initialization.seed + seed_shift,
MetaSizeInBits);
}
}
tools/profiler/src/gemm_operation_profiler.cu
+74 -39
View File
@@ -39,6 +39,7 @@
#include <vector>
#include "cutlass/core_io.h"
#include <cuda_runtime_api.h>
#include "cutlass/profiler/cublas_helpers.h"
#include "cutlass/profiler/gemm_operation_profiler.h"
@@ -46,7 +47,6 @@
#include "cutlass/library/singleton.h"
#include "cutlass/library/library.h"
#include "cutlass/library/handle.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -485,6 +485,17 @@ Status GemmOperationProfiler::initialize_workspace(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::Operation const* underlying_operation = operation;
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
@@ -496,12 +507,14 @@ Status GemmOperationProfiler::initialize_workspace(
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
bool is_sparse = operation_desc.tile_description.math_instruction.opcode_class == cutlass::library::OpcodeClassID::kSparseTensorOp;
// Compute the number of copies of the problem to avoid L2 camping.
if (!options.profiling.workspace_count) {
int64_t bytes = problem_.bytes(operation_desc);
if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
if (bytes < 3 * int64_t(options.device.properties[0].l2CacheSize)) {
gemm_workspace_.problem_count =
1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
1 + int((3 * int64_t(options.device.properties[0].l2CacheSize)) / bytes);
}
else {
gemm_workspace_.problem_count = 1;
@@ -514,7 +527,7 @@ Status GemmOperationProfiler::initialize_workspace(
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
int seed_shift = 0;
gemm_workspace_.A = device_context.allocate_tensor(
gemm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -522,10 +535,11 @@ Status GemmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.B = device_context.allocate_tensor(
gemm_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -533,10 +547,11 @@ Status GemmOperationProfiler::initialize_workspace(
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.C = device_context.allocate_tensor(
gemm_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -544,25 +559,30 @@ Status GemmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
gemm_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
}
@@ -580,7 +600,7 @@ Status GemmOperationProfiler::initialize_workspace(
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
/* Query device SM count to pass onto the kernel as an argument, where needed */
gemm_workspace_.arguments.sm_count = options.device.properties.multiProcessorCount;
gemm_workspace_.arguments.sm_count = options.device.properties[0].multiProcessorCount;
}
//
@@ -596,12 +616,34 @@ Status GemmOperationProfiler::initialize_workspace(
workspace_size = underlying_operation->get_device_workspace_size(&gemm_workspace_.configuration,
&gemm_workspace_.arguments);
if (is_sparse) {
// sparse gemm get_device_workspace_size() only return device workspace size per iteration
// Needs to multiply it w/ number of iteration
workspace_size *= gemm_workspace_.problem_count;
}
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
// Convert to structure sparse contents here.
if (is_sparse) {
uint8_t* profiler_workspaces[1];
profiler_workspaces[0] = reinterpret_cast<uint8_t*>(gemm_workspace_.A->data());
// Sparse operations have a different initialize interface.
// initialize_with_profiler_workspace converts mxk tensorA to compressed mxk/sp tensorA and the tensorE
auto modifiable_underlying_op = const_cast<library::Operation*>(underlying_operation);
status = modifiable_underlying_op->initialize_with_profiler_workspace(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data(),
profiler_workspaces,
gemm_workspace_.problem_count);
}
else {
status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
}
if (status != Status::kSuccess) {
return status;
}
@@ -821,26 +863,14 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
CublasLtCreate handle;
cublasStatus_t status = handle.get_cublaslt_create_status();
if (status != CUBLAS_STATUS_SUCCESS) {
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
return true;
}
std::vector<cublasGemmAlgo_t> algorithms;
detail::select_cublas_algorithms(
algorithms,
options,
gemm_desc);
if (algorithms.empty()) {
// no algorithm selected
return true;
}
//
// Initialize state
@@ -865,29 +895,34 @@ bool GemmOperationProfiler::verify_with_cublas_(
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
detail::cublasGemmExDispatcher gemm_op(
detail::cublasLtGemmExDispatcher gemm_op(
gemm_desc,
gemm_workspace_.configuration,
gemm_workspace_.arguments,
algorithms.front()
gemm_workspace_.arguments
);
gemm_op.initialize_cublaslt();
if(!gemm_op.get_cublaslt_algo(handle, AlgorithmMode::kDefault)){
return true;
}
if (gemm_op.status != Status::kSuccess) {
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
return true;
}
results_.back().status = Status::kSuccess;
status = gemm_op(handle);
// Handle errors
if (status != CUBLAS_STATUS_SUCCESS) {
std::cerr << "cublasLt Verification run failed with status : " << cublasLtGetStatusName(status) << "\n";
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
return true;
}
results_.back().status = Status::kSuccess;
//
// Verify results
//
@@ -930,9 +965,9 @@ bool GemmOperationProfiler::verify_with_reference_(
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem,
cutlass::library::NumericTypeID element_A,
cutlass::library::NumericTypeID element_B)
ProblemSpace::Problem const &problem,
cutlass::library::NumericTypeID element_A,
cutlass::library::NumericTypeID element_B)
{
library::GemmDescription const &gemm_desc =
static_cast<library::GemmDescription const &>(operation->description());
tools/profiler/src/operation_profiler.cu
+6 -6
View File
@@ -376,14 +376,14 @@ int OperationProfiler::profile_all(
std::cerr << " @ provider " << operation->description().provider
<< " != library::Provider::kCUTLASS\n";
}
if (options.device.compute_capability() < min_cc) {
if (options.device.compute_capability(0) < min_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability()
<< options.device.compute_capability(0)
<< " < min_cc " << min_cc << "\n";
}
if (options.device.compute_capability() > max_cc) {
if (options.device.compute_capability(0) > max_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability()
<< options.device.compute_capability(0)
<< " > max_cc " << max_cc << "\n";
}
#endif
@@ -391,8 +391,8 @@ int OperationProfiler::profile_all(
// Execute compatible cutlass operations if they satisfy the current device's compute capability
if (operation->description().kind == kind_ &&
operation->description().provider == library::Provider::kCUTLASS &&
options.device.compute_capability() >= min_cc &&
options.device.compute_capability() <= max_cc) {
options.device.compute_capability(0) >= min_cc &&
options.device.compute_capability(0) <= max_cc) {
std::string operation_name(operation->description().name);
// Filter kernels by name
tools/profiler/src/options.cu
+122 -62
View File
@@ -33,6 +33,7 @@
*/
#include <algorithm>
#include <set>
#include "cutlass/cutlass.h"
#include "cutlass/version.h"
@@ -55,45 +56,97 @@ static char const *end_of_line = "\n
Options::Device::Device(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("device", device, 0);
// Gets the number of devices for future validation
cudaError_t result;
result = cudaGetDeviceProperties(&properties, device);
result = cudaGetDeviceCount(&num_devices);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed for given device");
throw std::runtime_error("cudaGetNumDevices() failed");
}
result = cudaSetDevice(device);
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed for given device.");
}
// Permit overriding the compute capability
if (cmdline.check_cmd_line_flag("compute-capability")) {
int cc = compute_capability();
cmdline.get_cmd_line_argument("compute-capability", cc, cc);
properties.major = cc / 10;
properties.minor = cc % 10;
}
// Permit overriding the L2 cache capacity
if (cmdline.check_cmd_line_flag("llc-capacity")) {
int llc_capacity = 0;
cmdline.get_cmd_line_argument("llc-capacity", llc_capacity, 0);
if (llc_capacity >= 0) {
properties.l2CacheSize = (llc_capacity << 10);
// Gets the devices specified by the user
// This preserves the user specified order and checks for duplicates
{
std::vector<int> temp_device_list;
cmdline.get_cmd_line_arguments("devices", temp_device_list);
if (temp_device_list.empty()) {
temp_device_list.push_back(0);
}
{
std::set<int> temp_device_set;
for (int device : temp_device_list) {
auto res = temp_device_set.insert(device);
if (!res.second) {
throw std::runtime_error("Duplicate device specified: " +
std::to_string(device));
} else if (device > num_devices) {
throw std::runtime_error("Bad device ID: " +
std::to_string(device));
} else {
devices.push_back(device);
}
}
}
}
properties.resize(devices.size());
// Retrieves properties for all specified devices
for (size_t device_index = 0; device_index < devices.size(); device_index++) {
int device = devices[device_index];
result = cudaGetDeviceProperties(&properties[device_index], device);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed for given device");
}
// Check that all devices are the same
if (device_index > 0) {
if ((properties[device_index].major != properties[0].major) ||
(properties[device_index].minor != properties[0].minor)) {
throw std::runtime_error("All selected devices must have the same "
"compute capability");
}
if (properties[device_index].l2CacheSize != properties[0].l2CacheSize) {
throw std::runtime_error("All selected devices must have the same "
"L2 cache size");
}
if (properties[device_index].multiProcessorCount != properties[0].multiProcessorCount) {
throw std::runtime_error("All selected devices must have the same "
"SM count");
}
}
result = cudaSetDevice(device);
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed for given device.");
}
// Permit overriding the compute capability
if (cmdline.check_cmd_line_flag("compute-capability")) {
int cc = compute_capability(device_index);
cmdline.get_cmd_line_argument("compute-capability", cc, cc);
properties[device_index].major = cc / 10;
properties[device_index].minor = cc % 10;
}
// Permit overriding the L2 cache capacity
if (cmdline.check_cmd_line_flag("llc-capacity")) {
int llc_capacity = 0;
cmdline.get_cmd_line_argument("llc-capacity", llc_capacity, 0);
if (llc_capacity >= 0) {
properties[device_index].l2CacheSize = (llc_capacity << 10);
}
}
}
}
void Options::Device::print_usage(std::ostream &out) const {
out << "Device:\n"
<< " --device=<int> "
<< " CUDA Device ID\n\n";
<< " --devices=<int>,<int>,... "
<< " CUDA Device IDs\n\n";
int device_count = 0;
cudaError_t result = cudaGetDeviceCount(&device_count);
@@ -111,11 +164,11 @@ void Options::Device::print_usage(std::ostream &out) const {
break;
}
else {
out << " [" << idx << "] - "
<< prop.name << " - SM " << prop.major << "." << prop.minor << ", "
<< prop.multiProcessorCount << " SMs @ " << (prop.clockRate / 1000.0) << " MHz, "
out << " [" << idx << "] - "
<< prop.name << " - SM " << prop.major << "." << prop.minor << ", "
<< prop.multiProcessorCount << " SMs @ " << (prop.clockRate / 1000.0) << " MHz, "
<< "L2 cache: " << (prop.l2CacheSize >> 20) << " MB, Global Memory: " << (prop.totalGlobalMem >> 30) << " GB"
<< std::endl;
<< std::endl;
}
}
out << "\n";
@@ -133,15 +186,8 @@ void Options::Device::print_usage(std::ostream &out) const {
}
void Options::Device::print_device_info(std::ostream &out) const {
int num_devices;
cudaDeviceProp props;
cudaError_t result;
result = cudaGetDeviceCount(&num_devices);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetNumDevices() failed");
}
out << "Device Name,SM,CUDA Device ID,Phy Device ID" << std::endl;
@@ -165,14 +211,28 @@ void Options::Device::print_device_info(std::ostream &out) const {
void Options::Device::print_options(std::ostream &out, int indent) const {
out
<< indent_str(indent) << "device: " << device << "\n"
<< indent_str(indent) << "clock: " << int(double(properties.clockRate) / 1000.0) << "\n"
<< indent_str(indent) << "compute-capability: " << compute_capability() << "\n";
<< indent_str(indent) << "devices: ";
for (int device : devices) {
out << device << ',';
}
out
<< "\n"
<< indent_str(indent) << "clock: " << int(double(properties[0].clockRate) / 1000.0) << "\n"
<< indent_str(indent) << "compute-capability: " << compute_capability(0) << "\n";
}
/// Returns the device ID from a device index
int Options::Device::device_id(size_t device_index) const {
if (device_index > devices.size()) {
throw std::runtime_error("Out of bounds device index: " +
std::to_string(device_index));
}
return devices.at(device_index);
}
/// Returns the compute capability of the listed device (e.g. 61, 60, 70, 75)
int Options::Device::compute_capability() const {
return properties.major * 10 + properties.minor;
int Options::Device::compute_capability(int device_index) const {
return properties[device_index].major * 10 + properties[device_index].minor;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,10 +267,10 @@ Options::Initialization::Initialization(cutlass::CommandLine const &cmdline) {
else {
// profiler chosen data distribution (allowed to change based on numeric types)
fix_data_distribution = false;
// set uniform data distribution with range [-4, 4]
// set uniform data distribution with range [-4, 4]
data_distribution.set_uniform(-4, 4, 0);
}
}
@@ -248,10 +308,10 @@ void Options::Initialization::get_distribution(
};
// Initalize pnz values to a default value of 100%
dist.gaussian.pnz = 100.0;
dist.gaussian.pnzA = 100.0;
dist.gaussian.pnzB = 100.0;
dist.gaussian.pnzC = 100.0;
dist.gaussian.pnz = 1.0;
dist.gaussian.pnzA = 1.0;
dist.gaussian.pnzB = 1.0;
dist.gaussian.pnzC = 1.0;
using KeyValueVector = std::vector<std::pair<std::string, std::string> >;
@@ -335,7 +395,7 @@ Options::Library::Library(cutlass::CommandLine const &cmdline) {
std::string mode = "default";
cmdline.get_cmd_line_argument("library-algo-mode", mode);
algorithm_mode = from_string<AlgorithmMode>(mode);
}
}
if (cmdline.check_cmd_line_flag("library-algos")) {
@@ -353,7 +413,7 @@ Options::Library::Library(cutlass::CommandLine const &cmdline) {
}
else {
int algo;
std::stringstream ss;
std::stringstream ss;
ss << token;
ss >> algo;
@@ -396,12 +456,12 @@ void Options::Library::print_options(std::ostream &out, int indent) const {
Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("workspace-count", workspace_count, 0);
cmdline.get_cmd_line_argument("workspace-count", workspace_count, 0);
cmdline.get_cmd_line_argument("warmup-iterations", warmup_iterations, 10);
cmdline.get_cmd_line_argument("profiling-iterations", iterations, 100);
cmdline.get_cmd_line_argument("sleep-duration", sleep_duration, 50);
cmdline.get_cmd_line_argument("profiling-enabled", enabled, true);
if (cmdline.check_cmd_line_flag("providers")) {
std::vector<std::string> tokens;
@@ -416,7 +476,7 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
else {
providers.push_back(library::Provider::kCUTLASS);
providers.push_back(library::Provider::kCUBLAS);
providers.push_back(library::Provider::kCUDNN);
providers.push_back(library::Provider::kCUDNN);
}
}
@@ -480,7 +540,7 @@ size_t Options::Profiling::index(library::Provider provider) const {
/////////////////////////////////////////////////////////////////////////////////////////////////
Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("verification-enabled", enabled, true);
if (enabled) {
cmdline.get_cmd_line_argument("verification-required", required, false);
@@ -500,7 +560,7 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
}
if (cmdline.check_cmd_line_flag("verification-providers")) {
std::vector<std::string> tokens;
cmdline.get_cmd_line_arguments("verification-providers", tokens);
@@ -516,7 +576,7 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
else {
providers.push_back(library::Provider::kCUBLAS);
providers.push_back(library::Provider::kReferenceDevice);
providers.push_back(library::Provider::kCUDNN);
providers.push_back(library::Provider::kCUDNN);
}
}
@@ -583,11 +643,11 @@ size_t Options::Verification::index(library::Provider provider) const {
/////////////////////////////////////////////////////////////////////////////////////////////////
Options::Report::Report(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("append", append, false);
cmdline.get_cmd_line_argument("output", output_path);
cmdline.get_cmd_line_argument("junit-output", junit_output_path);
if (cmdline.check_cmd_line_flag("tags")) {
cmdline.get_cmd_line_argument_pairs("tags", pivot_tags);
}
@@ -687,11 +747,11 @@ Options::Options(cutlass::CommandLine const &cmdline):
device(cmdline),
initialization(cmdline),
library(cmdline),
profiling(cmdline),
verification(cmdline),
profiling(cmdline),
verification(cmdline),
report(cmdline),
about(cmdline) {
if (cmdline.check_cmd_line_flag("mode")) {
std::string token;
cmdline.get_cmd_line_argument("mode", token);
tools/profiler/src/performance_report.cpp
+21 -21
View File
@@ -94,7 +94,7 @@ PerformanceReport::PerformanceReport(
if (options_.report.append) {
std::ifstream test_output_file(op_file_name_);
if (test_output_file.is_open()) {
print_header = false;
test_output_file.close();
@@ -145,7 +145,7 @@ void PerformanceReport::append_result(PerformanceResult result) {
if (options_.report.verbose) {
std::cout << "\n";
print_result_pretty_(std::cout, result) << std::flush;
print_result_pretty_(std::cout, result) << std::flush;
}
if (junit_output_file_.is_open()) {
@@ -237,7 +237,7 @@ static const char *disposition_status_color(Disposition disposition) {
/// Prints the result in human readable form
std::ostream & PerformanceReport::print_result_pretty_(
std::ostream &out,
std::ostream &out,
PerformanceResult const &result,
bool use_shell_coloring) {
@@ -251,14 +251,14 @@ std::ostream & PerformanceReport::print_result_pretty_(
int column_idx = 0;
for (auto const & tag : options_.report.pivot_tags) {
out << (column_idx++ ? "," : "") << tag.first << ":" << tag.second;
}
}
out << "\n";
}
std::string shell_color_bright = use_shell_coloring ? SHELL_COLOR_BRIGHT() : "";
std::string shell_color_end = use_shell_coloring ? SHELL_COLOR_END() : "";
auto _disposition_status_color = [&](Disposition d) -> const char * {
auto _disposition_status_color = [&](Disposition d) -> const char * {
return use_shell_coloring ? disposition_status_color(d) : "";
};
@@ -277,7 +277,7 @@ std::ostream & PerformanceReport::print_result_pretty_(
static int const indent_spaces = 16;
for(auto & m : result.verification_map) {
out << std::right << std::setw(indent_spaces) << library::to_string(m.first, true) << ": " << to_string(m.second, true) << "\n";
out << std::right << std::setw(indent_spaces) << library::to_string(m.first, true) << ": " << to_string(m.second, true) << "\n";
}
}
@@ -287,7 +287,7 @@ std::ostream & PerformanceReport::print_result_pretty_(
int column_idx = 0;
for (auto const &arg : result.arguments) {
if (!arg.second.empty()) {
out << " --" << arg.first << "=" << arg.second;
out << " --" << arg.first << "=" << arg.second;
column_idx += int(4 + arg.first.size() + arg.second.size());
if (column_idx > 98) {
out << " \\\n ";
@@ -297,7 +297,7 @@ std::ostream & PerformanceReport::print_result_pretty_(
}
out << "\n\n";
out
out
<< " Bytes: " << result.bytes << " bytes\n"
<< " FLOPs: " << result.flops << " flops\n"
<< " FLOPs/Byte: " << (result.flops / result.bytes) << "\n\n";
@@ -325,7 +325,7 @@ std::ostream & PerformanceReport::print_csv_header_(
out << (column_idx++ ? "," : "") << tag.first;
}
out
out
<< (column_idx ? "," : "") << "Problem,Provider"
<< ",OperationKind,Operation,Disposition,Status";
@@ -333,7 +333,7 @@ std::ostream & PerformanceReport::print_csv_header_(
out << "," << arg_name;
}
out
out
<< ",Bytes"
<< ",Flops"
<< ",Flops/Byte"
@@ -347,7 +347,7 @@ std::ostream & PerformanceReport::print_csv_header_(
/// Print the result in CSV output
std::ostream & PerformanceReport::print_result_csv_(
std::ostream &out,
std::ostream &out,
PerformanceResult const &result) {
int column_idx = 0;
@@ -357,8 +357,8 @@ std::ostream & PerformanceReport::print_result_csv_(
out << (column_idx++ ? "," : "") << tag.second;
}
out
<< (column_idx ? "," : "")
out
<< (column_idx ? "," : "")
<< result.problem_index
<< "," << to_string(result.provider, true)
<< "," << to_string(result.op_kind)
@@ -370,7 +370,7 @@ std::ostream & PerformanceReport::print_result_csv_(
out << "," << arg.second;
}
out
out
<< "," << result.bytes
<< "," << result.flops
<< "," << result.flops / result.bytes
@@ -387,7 +387,7 @@ std::ostream & PerformanceReport::print_result_csv_(
else {
out << std::string(2
, ','
);
);
}
return out;
@@ -451,25 +451,25 @@ std::ostream & PerformanceReport::print_junit_result_(std::ostream &out, Perform
case Disposition::kNotSupported:
skipped = true;
break;
case Disposition::kPassed:
case Disposition::kPassed:
case Disposition::kNotVerified:
break;
case Disposition::kFailed:
case Disposition::kFailed:
case Disposition::kIncorrect:
failed = true;
failed = true;
break;
case Disposition::kInvalidProblem:
case Disposition::kInvalid:
error = true;
break;
};
if (skipped) {
out << "status=\"notrun\"";
} else {
out << "status=\"run\"";
}
out << ">" << std::endl;
if (failed) {
@@ -488,7 +488,7 @@ std::ostream & PerformanceReport::print_junit_result_(std::ostream &out, Perform
out << " </testcase>" << std::endl;
return out;
return out;
}
tools/profiler/src/rank_2k_operation_profiler.cu
+67 -47
View File
@@ -31,7 +31,7 @@
/* \file
\brief Execution environment
*/
#include <iostream>
@@ -54,7 +54,7 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
Rank2KOperationProfiler::Rank2KOperationProfiler(Options const &options):
Rank2KOperationProfiler::Rank2KOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kRank2K,
@@ -95,7 +95,7 @@ void Rank2KOperationProfiler::print_examples(std::ostream &out) const {
out << "\nExamples:\n\n"
<< "Profile a particular problem size Syrk kernel:\n"
<< " $ cutlass_profiler --operation=rank_2k --blas_mode=symmetric --n=1024 --k=128\n\n"
<< "Profile a particular problem size Herk kernel:\n"
<< " $ cutlass_profiler --operation=rank_2k --blas_mode=hermitian --n=1024 --k=128\n\n"
@@ -118,7 +118,7 @@ void Rank2KOperationProfiler::print_examples(std::ostream &out) const {
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=rank_2k --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to rank_2k kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=rank_2k \\ \n"
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -148,22 +148,22 @@ Status Rank2KOperationProfiler::RankKProblem::parse(
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
this->n = 1024;
}
if (!arg_as_int(this->k, "k", problem_space, problem)) {
// default value
this->k = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -187,29 +187,29 @@ Status Rank2KOperationProfiler::RankKProblem::parse(
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(this->n), int(this->k)}).front();
@@ -311,14 +311,14 @@ void Rank2KOperationProfiler::RankKProblem::initialize_result(
/// Extracts the problem dimensions
Status Rank2KOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::RankKDescription const &operation_desc =
library::RankKDescription const &operation_desc =
static_cast<library::RankKDescription const &>(operation->description());
if (operation_desc.rank_k_kind != library::RankKKind::kUniversal) {
@@ -326,7 +326,7 @@ Status Rank2KOperationProfiler::initialize_configuration(
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
@@ -350,14 +350,14 @@ Status Rank2KOperationProfiler::initialize_configuration(
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&rank_k_workspace_.configuration, &rank_k_workspace_.arguments);
}
/// Initializes the performance result
void Rank2KOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -365,7 +365,7 @@ void Rank2KOperationProfiler::initialize_result_(
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
@@ -380,19 +380,30 @@ void Rank2KOperationProfiler::initialize_result_(
/// Initializes workspace
Status Rank2KOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::RankKDescription const &operation_desc =
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::RankKDescription const &operation_desc =
static_cast<library::RankKDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
rank_k_workspace_.A = device_context.allocate_tensor(
rank_k_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -400,10 +411,11 @@ Status Rank2KOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.k)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
rank_k_workspace_.B = device_context.allocate_tensor(
rank_k_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -411,10 +423,11 @@ Status Rank2KOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.k)},
{int(problem_.ldb)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
rank_k_workspace_.C = device_context.allocate_tensor(
rank_k_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -422,23 +435,30 @@ Status Rank2KOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
rank_k_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
rank_k_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
rank_k_workspace_.Computed->copy_from_device(rank_k_workspace_.C->data());
@@ -487,7 +507,7 @@ Status Rank2KOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool Rank2KOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -516,7 +536,7 @@ bool Rank2KOperationProfiler::verify_cutlass(
//
results_.back().status = operation->run(
&rank_k_workspace_.arguments,
&rank_k_workspace_.arguments,
rank_k_workspace_.host_workspace.data(),
rank_k_workspace_.device_workspace.data());
@@ -564,8 +584,8 @@ bool Rank2KOperationProfiler::verify_cutlass(
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -591,7 +611,7 @@ bool Rank2KOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against references
bool Rank2KOperationProfiler::verify_with_cublas_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -601,13 +621,13 @@ bool Rank2KOperationProfiler::verify_with_cublas_(
#if CUTLASS_ENABLE_CUBLAS
library::RankKDescription const &rank_k_desc =
library::RankKDescription const &rank_k_desc =
static_cast<library::RankKDescription const &>(operation->description());
//
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
@@ -636,8 +656,8 @@ bool Rank2KOperationProfiler::verify_with_cublas_(
rank_k_workspace_.arguments.beta = problem_.beta.data();
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
detail::cublasRankKDispatcher rank_k_op(
rank_k_desc,
detail::cublasRankKDispatcher rank_k_op(
rank_k_desc,
rank_k_workspace_.configuration,
rank_k_workspace_.arguments
);
@@ -669,7 +689,7 @@ bool Rank2KOperationProfiler::verify_with_cublas_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
save_workspace(
@@ -694,7 +714,7 @@ bool Rank2KOperationProfiler::verify_with_cublas_(
/// Measures performance results
bool Rank2KOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
tools/profiler/src/rank_k_operation_profiler.cu
+66 -47
View File
@@ -31,7 +31,7 @@
/* \file
\brief Execution environment
*/
#include <iostream>
@@ -54,7 +54,7 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
RankKOperationProfiler::RankKOperationProfiler(Options const &options):
RankKOperationProfiler::RankKOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kRankK,
@@ -94,7 +94,7 @@ void RankKOperationProfiler::print_examples(std::ostream &out) const {
out << "\nExamples:\n\n"
<< "Profile a particular problem size Syrk kernel:\n"
<< " $ cutlass_profiler --operation=rank_k --blas_mode=symmetric --n=1024 --k=128\n\n"
<< "Profile a particular problem size Herk kernel:\n"
<< " $ cutlass_profiler --operation=rank_k --blas_mode=hermitian --n=1024 --k=128\n\n"
@@ -117,7 +117,7 @@ void RankKOperationProfiler::print_examples(std::ostream &out) const {
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=rank_k --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to rank_k kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=rank_k \\ \n"
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -147,22 +147,22 @@ Status RankKOperationProfiler::RankKProblem::parse(
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
this->n = 1024;
}
if (!arg_as_int(this->k, "k", problem_space, problem)) {
// default value
this->k = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -182,29 +182,29 @@ Status RankKOperationProfiler::RankKProblem::parse(
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(this->n), int(this->k)}).front();
@@ -252,7 +252,7 @@ int64_t RankKOperationProfiler::RankKProblem::flops(library::RankKDescription co
case library::MathOperationID::kMultiplyAddComplexFastF32:
flops_ *= 4;
break;
case library::MathOperationID::kMultiplyAddGaussianComplex:
flops_ *= 3;
break;
@@ -300,14 +300,14 @@ void RankKOperationProfiler::RankKProblem::initialize_result(
/// Extracts the problem dimensions
Status RankKOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::RankKDescription const &operation_desc =
library::RankKDescription const &operation_desc =
static_cast<library::RankKDescription const &>(operation->description());
if (operation_desc.rank_k_kind != library::RankKKind::kUniversal) {
@@ -315,7 +315,7 @@ Status RankKOperationProfiler::initialize_configuration(
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
@@ -337,14 +337,14 @@ Status RankKOperationProfiler::initialize_configuration(
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&rank_k_workspace_.configuration, &rank_k_workspace_.arguments);
}
/// Initializes the performance result
void RankKOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::RankKDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -352,7 +352,7 @@ void RankKOperationProfiler::initialize_result_(
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
@@ -368,7 +368,7 @@ void RankKOperationProfiler::initialize_result_(
case library::MathOperationID::kMultiplyAddComplex:
result.flops *= 4;
break;
case library::MathOperationID::kMultiplyAddComplexFastF32:
result.flops *= 4;
break;
@@ -380,19 +380,30 @@ void RankKOperationProfiler::initialize_result_(
/// Initializes workspace
Status RankKOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::RankKDescription const &operation_desc =
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::RankKDescription const &operation_desc =
static_cast<library::RankKDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
rank_k_workspace_.A = device_context.allocate_tensor(
rank_k_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -400,10 +411,11 @@ Status RankKOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.k)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
rank_k_workspace_.C = device_context.allocate_tensor(
rank_k_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -411,23 +423,30 @@ Status RankKOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
rank_k_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, //batch_count
0 // device_index
);
rank_k_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.n), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, //batch_count
0 // device_index
);
rank_k_workspace_.Computed->copy_from_device(rank_k_workspace_.C->data());
@@ -476,7 +495,7 @@ Status RankKOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool RankKOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -504,7 +523,7 @@ bool RankKOperationProfiler::verify_cutlass(
//
results_.back().status = operation->run(
&rank_k_workspace_.arguments,
&rank_k_workspace_.arguments,
rank_k_workspace_.host_workspace.data(),
rank_k_workspace_.device_workspace.data());
@@ -552,8 +571,8 @@ bool RankKOperationProfiler::verify_cutlass(
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -579,7 +598,7 @@ bool RankKOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against references
bool RankKOperationProfiler::verify_with_cublas_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -589,13 +608,13 @@ bool RankKOperationProfiler::verify_with_cublas_(
#if CUTLASS_ENABLE_CUBLAS
library::RankKDescription const &rank_k_desc =
library::RankKDescription const &rank_k_desc =
static_cast<library::RankKDescription const &>(operation->description());
//
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
@@ -623,8 +642,8 @@ bool RankKOperationProfiler::verify_with_cublas_(
rank_k_workspace_.arguments.beta = problem_.beta.data();
rank_k_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
detail::cublasRankKDispatcher rank_k_op(
rank_k_desc,
detail::cublasRankKDispatcher rank_k_op(
rank_k_desc,
rank_k_workspace_.configuration,
rank_k_workspace_.arguments
);
@@ -656,7 +675,7 @@ bool RankKOperationProfiler::verify_with_cublas_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
save_workspace(
@@ -681,7 +700,7 @@ bool RankKOperationProfiler::verify_with_cublas_(
/// Measures performance results
bool RankKOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
tools/profiler/src/sparse_gemm_operation_profiler.cu
+72 -51
View File
@@ -51,23 +51,23 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
SparseGemmOperationProfiler::SparseGemmOperationProfiler(Options const &options):
SparseGemmOperationProfiler::SparseGemmOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kSparseGemm,
{
{ArgumentTypeID::kEnumerated, {"gemm_kind"}, "Variant of GEMM (e.g. sparse, ...)"},
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
{ArgumentTypeID::kTensor, {"E"}, "Tensor storing the E operand"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
{ArgumentTypeID::kInteger, {"split_k_slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count"}, "Number of GEMMs computed in one batch"},
{ArgumentTypeID::kEnumerated, {"gemm_kind"}, "Variant of GEMM (e.g. sparse, ...)"},
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
{ArgumentTypeID::kTensor, {"E"}, "Tensor storing the E operand"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
{ArgumentTypeID::kInteger, {"split_k_slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count"}, "Number of GEMMs computed in one batch"},
}
) {
@@ -109,7 +109,7 @@ void SparseGemmOperationProfiler::print_examples(std::ostream &out) const {
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=SparseGemm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to gemm kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=SparseGemm \\ \n"
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -125,7 +125,7 @@ Status SparseGemmOperationProfiler::SparseGemmProblem::parse(
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
this->m = 1024;
@@ -135,17 +135,17 @@ Status SparseGemmOperationProfiler::SparseGemmProblem::parse(
// default value
this->n = 1024;
}
if (!arg_as_int(this->k, "k", problem_space, problem)) {
// default value
this->k = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -168,24 +168,24 @@ Status SparseGemmOperationProfiler::SparseGemmProblem::parse(
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
@@ -252,14 +252,14 @@ void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
/// Extracts the problem dimensions
Status SparseGemmOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SparseGemmDescription const &operation_desc =
library::SparseGemmDescription const &operation_desc =
static_cast<library::SparseGemmDescription const &>(operation->description());
if (operation_desc.gemm_kind != library::GemmKind::kSparse) {
@@ -291,14 +291,14 @@ Status SparseGemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
}
/// Initializes the performance result
void SparseGemmOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -308,7 +308,7 @@ void SparseGemmOperationProfiler::initialize_result_(
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
// Input bytes read and Output bytes written for the gemm problem
@@ -337,19 +337,30 @@ void SparseGemmOperationProfiler::initialize_result_(
/// Initializes workspace
Status SparseGemmOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SparseGemmDescription const &operation_desc =
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::SparseGemmDescription const &operation_desc =
static_cast<library::SparseGemmDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
gemm_workspace_.A = device_context.allocate_tensor(
gemm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -357,10 +368,11 @@ Status SparseGemmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.k) / int(problem_.sparse)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.B = device_context.allocate_tensor(
gemm_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -368,10 +380,11 @@ Status SparseGemmOperationProfiler::initialize_workspace(
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.C = device_context.allocate_tensor(
gemm_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -379,18 +392,22 @@ Status SparseGemmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
gemm_workspace_.E = device_context.allocate_sparsemeta_tensor(
gemm_workspace_.E = device_context.allocate_and_initialize_sparsemeta_tensor(
options,
"E",
operation_desc.E.element,
@@ -399,15 +416,19 @@ Status SparseGemmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.k) / int(problem_.sparse) / int(problem_.elements_per_128b)},
{int(problem_.lde)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
gemm_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
@@ -456,7 +477,7 @@ Status SparseGemmOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool SparseGemmOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -486,7 +507,7 @@ bool SparseGemmOperationProfiler::verify_cutlass(
//
results_.back().status = operation->run(
&gemm_workspace_.arguments,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
@@ -510,7 +531,7 @@ bool SparseGemmOperationProfiler::verify_cutlass(
if (options.verification.enabled) {
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
@@ -537,7 +558,7 @@ bool SparseGemmOperationProfiler::verify_cutlass(
/// Measures performance results
bool SparseGemmOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -565,7 +586,7 @@ bool SparseGemmOperationProfiler::profile(
gemm_workspace_.device_workspace.data()
);
}
return true;
}
tools/profiler/src/symm_operation_profiler.cu
+74 -53
View File
@@ -31,7 +31,7 @@
/* \file
\brief Execution environment
*/
#include <iostream>
@@ -54,7 +54,7 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
SymmOperationProfiler::SymmOperationProfiler(Options const &options):
SymmOperationProfiler::SymmOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kSymm,
@@ -96,7 +96,7 @@ void SymmOperationProfiler::print_examples(std::ostream &out) const {
out << "\nExamples:\n\n"
<< "Profile a particular problem size SYMM kernel:\n"
<< " $ cutlass_profiler --operation=Symm --blas_mode=symmetric --m=1024 --n=128\n\n"
<< "Profile a particular problem size HEMM kernel:\n"
<< " $ cutlass_profiler --operation=Symm --blas_mode=hermitian --m=1024 --n=128\n\n"
@@ -122,7 +122,7 @@ void SymmOperationProfiler::print_examples(std::ostream &out) const {
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=Symm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to symm kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=Symm \\ \n"
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -152,22 +152,22 @@ Status SymmOperationProfiler::SymmProblem::parse(
library::SymmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
this->m = 1024;
}
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
this->n = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -191,29 +191,29 @@ Status SymmOperationProfiler::SymmProblem::parse(
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
if (operation_desc.side_mode == SideMode::kLeft) {
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(this->m), int(this->m)}).front();
@@ -240,12 +240,12 @@ int64_t SymmOperationProfiler::SymmProblem::bytes(library::SymmDescription const
if (operation_desc.side_mode == SideMode::kLeft) {
bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * (m + 1) / 2 +
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
} else if (operation_desc.side_mode == SideMode::kRight) {
bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * n / 8) * (n + 1) / 2 +
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
int64_t(library::sizeof_bits(operation_desc.B.element) * m / 8) * n +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
}
// Set is_beta_zero true if beta is zero
@@ -277,7 +277,7 @@ int64_t SymmOperationProfiler::SymmProblem::flops(library::SymmDescription const
case library::MathOperationID::kMultiplyAddComplex:
flops_ *= 4;
break;
case library::MathOperationID::kMultiplyAddComplexFastF32:
flops_ *= 4;
break;
@@ -334,14 +334,14 @@ void SymmOperationProfiler::SymmProblem::initialize_result(
/// Extracts the problem dimensions
Status SymmOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SymmDescription const &operation_desc =
library::SymmDescription const &operation_desc =
static_cast<library::SymmDescription const &>(operation->description());
if (operation_desc.symm_kind != library::SymmKind::kUniversal) {
@@ -349,14 +349,14 @@ Status SymmOperationProfiler::initialize_configuration(
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
symm_workspace_.configuration.problem_size.m() = int(problem_.m);
symm_workspace_.configuration.problem_size.n() = int(problem_.n);
symm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
symm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
? int(problem_.m) : int(problem_.n);
symm_workspace_.configuration.lda = problem_.lda;
symm_workspace_.configuration.ldb = problem_.ldb;
@@ -374,14 +374,14 @@ Status SymmOperationProfiler::initialize_configuration(
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&symm_workspace_.configuration, &symm_workspace_.arguments);
}
/// Initializes the performance result
void SymmOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::SymmDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -389,7 +389,7 @@ void SymmOperationProfiler::initialize_result_(
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
@@ -404,20 +404,31 @@ void SymmOperationProfiler::initialize_result_(
/// Initializes workspace
Status SymmOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SymmDescription const &operation_desc =
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::SymmDescription const &operation_desc =
static_cast<library::SymmDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
if (operation_desc.side_mode == SideMode::kLeft) {
symm_workspace_.A = device_context.allocate_tensor(
symm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -425,10 +436,11 @@ Status SymmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.m)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
} else if (operation_desc.side_mode == SideMode::kRight) {
symm_workspace_.A = device_context.allocate_tensor(
symm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -436,11 +448,12 @@ Status SymmOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.n)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
}
symm_workspace_.B = device_context.allocate_tensor(
symm_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -448,10 +461,11 @@ Status SymmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.n)},
{int(problem_.ldb)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
symm_workspace_.C = device_context.allocate_tensor(
symm_workspace_.C = device_context.allocate_and_initialize_tensor(
options,
"C",
operation_desc.C.element,
@@ -459,23 +473,30 @@ Status SymmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
symm_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
symm_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
1, // batch_count
0 // device_index
);
symm_workspace_.Computed->copy_from_device(symm_workspace_.C->data());
@@ -524,7 +545,7 @@ Status SymmOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool SymmOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -553,7 +574,7 @@ bool SymmOperationProfiler::verify_cutlass(
//
results_.back().status = operation->run(
&symm_workspace_.arguments,
&symm_workspace_.arguments,
symm_workspace_.host_workspace.data(),
symm_workspace_.device_workspace.data());
@@ -601,8 +622,8 @@ bool SymmOperationProfiler::verify_cutlass(
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -628,7 +649,7 @@ bool SymmOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against references
bool SymmOperationProfiler::verify_with_cublas_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -638,13 +659,13 @@ bool SymmOperationProfiler::verify_with_cublas_(
#if CUTLASS_ENABLE_CUBLAS
library::SymmDescription const &symm_desc =
library::SymmDescription const &symm_desc =
static_cast<library::SymmDescription const &>(operation->description());
//
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
@@ -673,8 +694,8 @@ bool SymmOperationProfiler::verify_with_cublas_(
symm_workspace_.arguments.beta = problem_.beta.data();
symm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
detail::cublasSymmDispatcher symm_op(
symm_desc,
detail::cublasSymmDispatcher symm_op(
symm_desc,
symm_workspace_.configuration,
symm_workspace_.arguments
);
@@ -706,7 +727,7 @@ bool SymmOperationProfiler::verify_with_cublas_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
save_workspace(
@@ -731,7 +752,7 @@ bool SymmOperationProfiler::verify_with_cublas_(
/// Measures performance results
bool SymmOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
tools/profiler/src/trmm_operation_profiler.cu
+74 -54
View File
@@ -31,7 +31,7 @@
/* \file
\brief Execution environment
*/
#include <iostream>
@@ -54,7 +54,7 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
TrmmOperationProfiler::TrmmOperationProfiler(Options const &options):
TrmmOperationProfiler::TrmmOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kTrmm,
@@ -113,7 +113,7 @@ void TrmmOperationProfiler::print_examples(std::ostream &out) const {
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=Trmm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to trmm kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=Trmm \\ \n"
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
@@ -143,22 +143,22 @@ Status TrmmOperationProfiler::TrmmProblem::parse(
library::TrmmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
this->m = 1024;
}
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
this->n = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -182,29 +182,29 @@ Status TrmmOperationProfiler::TrmmProblem::parse(
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
if (operation_desc.side_mode == SideMode::kLeft) {
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(this->m), int(this->m)}).front();
@@ -265,14 +265,14 @@ void TrmmOperationProfiler::TrmmProblem::initialize_result(
/// Extracts the problem dimensions
Status TrmmOperationProfiler::initialize_configuration(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::TrmmDescription const &operation_desc =
library::TrmmDescription const &operation_desc =
static_cast<library::TrmmDescription const &>(operation->description());
if (operation_desc.trmm_kind != library::TrmmKind::kUniversal) {
@@ -280,14 +280,14 @@ Status TrmmOperationProfiler::initialize_configuration(
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
trmm_workspace_.configuration.problem_size.m() = int(problem_.m);
trmm_workspace_.configuration.problem_size.n() = int(problem_.n);
trmm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
trmm_workspace_.configuration.problem_size.k() = (operation_desc.side_mode == SideMode::kLeft)
? int(problem_.m) : int(problem_.n);
trmm_workspace_.configuration.lda = problem_.lda;
trmm_workspace_.configuration.ldb = problem_.ldb;
@@ -303,14 +303,14 @@ Status TrmmOperationProfiler::initialize_configuration(
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&trmm_workspace_.configuration, &trmm_workspace_.arguments);
}
/// Initializes the performance result
void TrmmOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
Options const &options,
library::TrmmDescription const &operation_desc,
ProblemSpace const &problem_space) {
@@ -318,30 +318,30 @@ void TrmmOperationProfiler::initialize_result_(
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
if (operation_desc.side_mode == SideMode::kLeft) {
// Input bytes read and Output bytes written for the trmm problem
result.bytes =
result.bytes =
// Half matrix including the diagonal will have (M*(M+1))/2 elements
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) * (problem_.m + 1) / 2 +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
int64_t(library::sizeof_bits(operation_desc.D.element) * problem_.m / 8) * problem_.n;
} else if (operation_desc.side_mode == SideMode::kRight) {
// Input bytes read and Output bytes written for the trmm problem
result.bytes =
result.bytes =
// Half matrix including the diagonal will have (N*(N+1))/2 elements
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.n / 8) * (problem_.n + 1) / 2 +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.m / 8) * problem_.n +
int64_t(library::sizeof_bits(operation_desc.D.element) * problem_.m / 8) * problem_.n;
}
// FLOPs = 2 * [ ( M * (M+1)/2 * N ) ] // Beta is zero
result.flops = problem_.m * (problem_.m + 1) * problem_.n;
result.runtime = 0;
// complex-valued support
@@ -349,11 +349,11 @@ void TrmmOperationProfiler::initialize_result_(
case library::MathOperationID::kMultiplyAddComplex:
result.flops *= 4;
break;
case library::MathOperationID::kMultiplyAddComplexFastF32:
result.flops *= 4;
break;
default: break;
}
@@ -361,20 +361,31 @@ void TrmmOperationProfiler::initialize_result_(
/// Initializes workspace
Status TrmmOperationProfiler::initialize_workspace(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::TrmmDescription const &operation_desc =
if (options.device.devices.size() != 1) {
throw std::runtime_error("This operation profiler only supports a single "
"device.");
}
cudaError_t result;
result = cudaSetDevice(options.device.device_id(0));
if (result != cudaSuccess) {
throw std::runtime_error("cudaSetDevice() failed.");
}
library::TrmmDescription const &operation_desc =
static_cast<library::TrmmDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
int seed_shift = 0;
if (operation_desc.side_mode == SideMode::kLeft) {
trmm_workspace_.A = device_context.allocate_tensor(
trmm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -382,10 +393,11 @@ Status TrmmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.m)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
} else if (operation_desc.side_mode == SideMode::kRight) {
trmm_workspace_.A = device_context.allocate_tensor(
trmm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
@@ -393,11 +405,12 @@ Status TrmmOperationProfiler::initialize_workspace(
{int(problem_.n), int(problem_.n)},
{int(problem_.lda)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
}
trmm_workspace_.B = device_context.allocate_tensor(
trmm_workspace_.B = device_context.allocate_and_initialize_tensor(
options,
"B",
operation_desc.B.element,
@@ -405,23 +418,30 @@ Status TrmmOperationProfiler::initialize_workspace(
{int(problem_.m), int(problem_.n)},
{int(problem_.ldb)},
1, // batch_count
seed_shift++
seed_shift++,
0 // device_index
);
trmm_workspace_.Computed = device_context.allocate_tensor(
options,
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldd)}
{int(problem_.ldd)},
1, // batch_count
0 // device_index
);
trmm_workspace_.Reference = device_context.allocate_tensor(
options,
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldd)}
{int(problem_.ldd)},
1, // batch_count
0 // device_index
);
}
@@ -467,7 +487,7 @@ Status TrmmOperationProfiler::initialize_workspace(
/// Verifies CUTLASS against references
bool TrmmOperationProfiler::verify_cutlass(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -495,7 +515,7 @@ bool TrmmOperationProfiler::verify_cutlass(
//
results_.back().status = operation->run(
&trmm_workspace_.arguments,
&trmm_workspace_.arguments,
trmm_workspace_.host_workspace.data(),
trmm_workspace_.device_workspace.data());
@@ -543,8 +563,8 @@ bool TrmmOperationProfiler::verify_cutlass(
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
// Update disposition to worst case verification outcome among all
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
@@ -570,7 +590,7 @@ bool TrmmOperationProfiler::verify_cutlass(
/// Verifies CUTLASS against references
bool TrmmOperationProfiler::verify_with_cublas_(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
@@ -580,13 +600,13 @@ bool TrmmOperationProfiler::verify_with_cublas_(
#if CUTLASS_ENABLE_CUBLAS
library::TrmmDescription const &trmm_desc =
library::TrmmDescription const &trmm_desc =
static_cast<library::TrmmDescription const &>(operation->description());
//
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
@@ -614,8 +634,8 @@ bool TrmmOperationProfiler::verify_with_cublas_(
trmm_workspace_.arguments.beta = problem_.beta.data();
trmm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
detail::cublasTrmmDispatcher trmm_op(
trmm_desc,
detail::cublasTrmmDispatcher trmm_op(
trmm_desc,
trmm_workspace_.configuration,
trmm_workspace_.arguments
);
@@ -646,7 +666,7 @@ bool TrmmOperationProfiler::verify_with_cublas_(
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
save_workspace(
@@ -671,7 +691,7 @@ bool TrmmOperationProfiler::verify_with_cublas_(
/// Measures performance results
bool TrmmOperationProfiler::profile(
Options const &options,
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,