leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

CUTLASS 3.5.0 (#1411)

Browse Source
This commit is contained in:
Vijay Thakkar
2024-03-19 17:51:04 -04:00
committed by GitHub
parent ffa34e7075
commit 629f4653c3
468 changed files with 48729 additions and 7252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/cutlass_profiler.cu
+5 -6
View File
@@ -42,8 +42,8 @@
#include "cutlass/profiler/rank_2k_operation_profiler.h"
#include "cutlass/profiler/trmm_operation_profiler.h"
#include "cutlass/profiler/symm_operation_profiler.h"
#include "cutlass/profiler/conv2d_operation_profiler.h"
#include "cutlass/profiler/conv3d_operation_profiler.h"
#include "cutlass/profiler/conv2d_operation_profiler.h"
#include "cutlass/profiler/conv3d_operation_profiler.h"
#include "cutlass/profiler/sparse_gemm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -55,7 +55,7 @@ namespace profiler {
CutlassProfiler::CutlassProfiler(
Options const &options
):
):
options_(options) {
operation_profilers_.emplace_back(new GemmOperationProfiler(options));
@@ -145,7 +145,6 @@ int CutlassProfiler::profile_() {
int result = 0;
DeviceContext device_context;
// For all profilers
for (auto & profiler : operation_profilers_) {
@@ -193,8 +192,8 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
<< " $ cutlass_profiler --operation=RankK --help\n\n"
<< " $ cutlass_profiler --operation=Trmm --help\n\n"
<< " $ cutlass_profiler --operation=Symm --help\n\n"
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
<< " $ cutlass_profiler --operation=SparseGemm --help\n\n"
;
}
tools/profiler/src/gemm_operation_profiler.cu
+3 -4
View File
@@ -36,6 +36,7 @@
#include <stdexcept>
#include <iomanip>
#include <ios>
#include <vector>
#include "cutlass/core_io.h"
@@ -167,7 +168,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
// default value
this->k = 1024;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
// default value
this->split_k_mode = library::SplitKMode::kSerial;
@@ -421,6 +422,7 @@ void GemmOperationProfiler::initialize_result_(
bool GemmOperationProfiler::initialize_reduction_configuration_(
library::Operation const *operation,
ProblemSpace::Problem const &problem) {
library::GemmDescription const &gemm_desc =
static_cast<library::GemmDescription const&>(operation->description());
@@ -577,7 +579,6 @@ Status GemmOperationProfiler::initialize_workspace(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
if (options.execution_mode != ExecutionMode::kDryRun) {
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.host_workspace.resize(workspace_size, 0);
@@ -620,7 +621,6 @@ Status GemmOperationProfiler::initialize_workspace(
results_.back().verification_map[provider] = Disposition::kNotRun;
}
}
return status;
}
@@ -794,7 +794,6 @@ bool GemmOperationProfiler::verify_with_cublas_(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
#if CUTLASS_ENABLE_CUBLAS
library::GemmDescription const &gemm_desc =
tools/profiler/src/operation_profiler.cu
+133 -36
View File
@@ -51,6 +51,8 @@
#include "cutlass/profiler/operation_profiler.h"
#include "cutlass/profiler/gpu_timer.h"
#include "cutlass/trace.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -66,7 +68,7 @@ OperationProfiler::OperationProfiler(
library::OperationKind kind,
ArgumentDescriptionVector const &arguments,
ProviderVector const & verification_providers
):
):
kind_(kind), arguments_(arguments) {
ArgumentDescriptionVector tile_description_arguments{
@@ -93,8 +95,8 @@ OperationProfiler::OperationProfiler(
for (auto provider : verification_providers) {
if (std::find(
options.verification.providers.begin(),
options.verification.providers.end(),
options.verification.providers.begin(),
options.verification.providers.end(),
provider) != options.verification.providers.end()) {
verification_providers_.push_back(provider);
@@ -118,14 +120,14 @@ void OperationProfiler::print_usage(std::ostream &out) const {
size_t const kAliasStart = 10;
size_t columns = 0;
std::string type_str = to_string(desc.type);
columns += type_str.size();
out << " [" << type_str << "]";
if (columns < kAliasStart) {
out << std::string(kAliasStart - columns, ' ');
out << std::string(kAliasStart - columns, ' ');
}
columns = 0;
@@ -161,7 +163,6 @@ bool OperationProfiler::satisfies(
return false;
}
}
int64_t int_value;
if (arg_as_int(int_value, "inst_m", problem_space, problem)) {
@@ -252,14 +253,79 @@ bool OperationProfiler::satisfies(
return true;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::ostream& operator<<(std::ostream& out, library::Provider provider) {
if (provider == library::Provider::kNone) {
out << "kNone";
}
else if (provider == library::Provider::kCUTLASS) {
out << "kCUTLASS";
}
else if (provider == library::Provider::kReferenceHost) {
out << "kReferenceHost";
}
else if (provider == library::Provider::kReferenceDevice) {
out << "kReferenceDevice";
}
else if (provider == library::Provider::kCUBLAS) {
out << "kCUBLAS";
}
else if (provider == library::Provider::kCUDNN) {
out << "kCUDNN";
}
else {
out << "kInvalid";
}
return out;
}
std::ostream& operator<<(std::ostream& out, library::OperationKind provider) {
if (provider == library::OperationKind::kGemm) {
out << "kGemm";
}
else if (provider == library::OperationKind::kRankK) {
out << "kRankK";
}
else if (provider == library::OperationKind::kRank2K) {
out << "kRank2K";
}
else if (provider == library::OperationKind::kTrmm) {
out << "kTrmm";
}
else if (provider == library::OperationKind::kSymm) {
out << "kSymm";
}
else if (provider == library::OperationKind::kConv2d) {
out << "kConv2d";
}
else if (provider == library::OperationKind::kConv3d) {
out << "kConv3d";
}
else if (provider == library::OperationKind::kEqGemm) {
out << "kEqGemm";
}
else if (provider == library::OperationKind::kSparseGemm) {
out << "kSparseGemm";
}
else if (provider == library::OperationKind::kReduction) {
out << "kReduction";
}
else {
out << "kInvalid";
}
return out;
}
#endif // defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
/// Entry point to profile all operations in the manifest
int OperationProfiler::profile_all(
Options const &options,
library::Manifest const &manifest,
Options const &options,
library::Manifest const &manifest,
DeviceContext &device_context) {
ProblemSpace problem_space(arguments_, options.cmdline);
// 1. Construct performance report
@@ -282,13 +348,45 @@ int OperationProfiler::profile_all(
for (auto const& operation_ptr : manifest) {
library::Operation const *operation = operation_ptr.get();
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " Operation: " << typeid(*operation).name() << "\n"
<< " name: " << operation->description().name << "\n"
<< " kind: " << operation->description().kind << "\n"
<< " provider: " << operation->description().provider << "\n";
#endif // CUTLASS_DEBUG_TRACE_LEVEL
auto min_cc = operation->description().tile_description.minimum_compute_capability;
auto max_cc = operation->description().tile_description.maximum_compute_capability;
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
std::cerr << " min_cc: " << min_cc << "\n";
std::cerr << " max_cc: " << min_cc << "\n";
#endif
// Clear named allocations
device_context.free();
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && (CUTLASS_DEBUG_TRACE_LEVEL > 1)
if (operation->description().kind != kind_) {
std::cerr << " @ kind " << operation->description().kind
<< " != kind_ " << kind_ << "\n";
}
if (operation->description().provider != library::Provider::kCUTLASS) {
std::cerr << " @ provider " << operation->description().provider
<< " != library::Provider::kCUTLASS\n";
}
if (options.device.compute_capability() < min_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability()
<< " < min_cc " << min_cc << "\n";
}
if (options.device.compute_capability() > max_cc) {
std::cerr << " @ compute_capability "
<< options.device.compute_capability()
<< " > max_cc " << max_cc << "\n";
}
#endif
// Execute compatible cutlass operations if they satisfy the current device's compute capability
if (operation->description().kind == kind_ &&
operation->description().provider == library::Provider::kCUTLASS &&
@@ -296,17 +394,16 @@ int OperationProfiler::profile_all(
options.device.compute_capability() <= max_cc) {
std::string operation_name(operation->description().name);
// Filter kernels by name
bool filtered_by_name = options.operation_names.empty();
if (!filtered_by_name) {
for (auto const & op_name : options.operation_names) {
if (find_string_matches_(op_name, operation_name)) {
filtered_by_name = true;
break;
}
}
}
}
for (auto const & op_name : options.excluded_operation_names) {
@@ -333,10 +430,10 @@ int OperationProfiler::profile_all(
problem);
if (status == Status::kErrorInternal) {
// If there was an internal error, consume the CUDA error and move to the next operation.
(void)cudaGetLastError();
report.append_results(results_);
continue;
}
@@ -385,9 +482,9 @@ int OperationProfiler::profile_all(
continue_profiling = this->verify_cutlass(
options,
report,
device_context,
operation,
report,
device_context,
operation,
problem_space,
problem);
@@ -419,10 +516,10 @@ int OperationProfiler::profile_all(
if (continue_profiling && options.profiling.enabled) {
continue_profiling = this->profile(
options,
report,
device_context,
operation,
options,
report,
device_context,
operation,
problem_space,
problem);
}
@@ -459,7 +556,7 @@ void OperationProfiler::sleep(int sleep_duration) {
SleepEx(sleep_duration, false);
#else
// sleep not supported
#endif
#endif
}
}
@@ -485,7 +582,7 @@ Disposition OperationProfiler::compare_tensors(
// bit-level equality
passed = DeviceAllocation::block_compare_equal(
experimental.type(),
experimental.type(),
experimental.data(),
reference.data(),
count);
@@ -494,7 +591,7 @@ Disposition OperationProfiler::compare_tensors(
// relative error function
passed = DeviceAllocation::block_compare_relatively_equal(
experimental.type(),
experimental.type(),
experimental.data(),
reference.data(),
count,
@@ -516,7 +613,7 @@ void OperationProfiler::save_workspace(
for (auto const & named_allocation : device_context) {
DeviceAllocation *allocation = named_allocation.second;
std::stringstream filename;
filename << desc.name << "_" << library::to_string(provider) << "_";
@@ -535,7 +632,7 @@ void OperationProfiler::save_workspace(
if (options.report.verbose) {
std::cout << "wrote '" << filename.str() << "'" << std::endl;
}
}
}
}
@@ -575,7 +672,7 @@ Status OperationProfiler::profile_cutlass_(
return status;
}
}
//
// Initialize GPU timer
//
@@ -590,7 +687,7 @@ Status OperationProfiler::profile_cutlass_(
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
status = operation->run(
arguments,
host_workspace,
@@ -610,7 +707,7 @@ Status OperationProfiler::profile_cutlass_(
//
// Update performance result
//
runtime = timer.duration(iteration);
return status;
@@ -618,7 +715,7 @@ Status OperationProfiler::profile_cutlass_(
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Sets operation description
/// Sets operation description
void OperationProfiler::initialize_result_(
PerformanceResult &result,
library::OperationDescription const &operation_desc,
@@ -657,7 +754,7 @@ void OperationProfiler::set_argument(
result.arguments.at(problem_space.argument_index(name)) = make_pair(std::string(name), value);
}
void OperationProfiler::set_argument(
void OperationProfiler::set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
@@ -669,12 +766,12 @@ void OperationProfiler::set_argument(
/// finds string matches filter_string in operation_name
bool OperationProfiler::find_string_matches_(
std::string const &filter_string,
std::string const &filter_string,
std::string const &operation_name) {
// Returns true if all substrings appear in the operation_name in order
// Split filter_string of the format "gemm*f32*nt" to tokens ["gemm", "f32", "nt"]
std::string item;
std::string item;
std::istringstream iss(filter_string);
std::vector<std::string> filter_tokens;
while (std::getline(iss, item, '*')) {
@@ -692,7 +789,7 @@ bool OperationProfiler::find_string_matches_(
return false;
}
}
start += (idx + token.length());
start += (idx + token.length());
}
// All tokens in filter_string found in operation_name
tools/profiler/src/symm_operation_profiler.cu
+1 -1
View File
@@ -234,7 +234,7 @@ Status SymmOperationProfiler::SymmProblem::parse(
/// Total number of bytes loaded
int64_t SymmOperationProfiler::SymmProblem::bytes(library::SymmDescription const &operation_desc) const {
int64_t bytes;
int64_t bytes = 0;
// Input bytes read and Output bytes written for the gemm problem
// Half matrix including the diagonal will have (X*(X+1))/2 elements
if (operation_desc.side_mode == SideMode::kLeft) {