leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

CUTLASS 3.8 Release (#2059)

Browse Source 
* CUTLASS 3.8 Release

* update

* Update README.md

* Revert "Update README.md"

This reverts commit b353e36fe83e0815f99b44e46c0c95494c44726b.

* update

* update

---------

Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
mihir-awatramani
2025-01-25 02:44:06 -05:00
committed by GitHub
co-authored by Haicheng Wu Haicheng Wu
parent 9eb01fa0b0
commit 389e493055
290 changed files with 91222 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
+1371
View File
File diff suppressed because it is too large Load Diff
tools/profiler/src/cutlass_profiler.cu
+3
View File
@@ -38,6 +38,7 @@
// Profiler includes
#include "cutlass/profiler/cutlass_profiler.h"
#include "cutlass/profiler/gemm_operation_profiler.h"
#include "cutlass/profiler/block_scaled_gemm_operation_profiler.h"
#include "cutlass/profiler/rank_k_operation_profiler.h"
#include "cutlass/profiler/rank_2k_operation_profiler.h"
#include "cutlass/profiler/trmm_operation_profiler.h"
@@ -60,6 +61,8 @@ CutlassProfiler::CutlassProfiler(
operation_profilers_.emplace_back(new GemmOperationProfiler(options));
operation_profilers_.emplace_back(new BlockScaledGemmOperationProfiler(options));
operation_profilers_.emplace_back(new SparseGemmOperationProfiler(options));
operation_profilers_.emplace_back(new Conv2dOperationProfiler(options));
tools/profiler/src/device_allocation.cu
+325
View File
@@ -616,6 +616,48 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::device::BlockFillRandom<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::device::BlockFillRandom<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::device::BlockFillRandom<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::device::BlockFillRandom<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::device::BlockFillRandom<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::device::BlockFillRandom<double>(
reinterpret_cast<double *>(pointer_),
@@ -771,6 +813,50 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFillRandom<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillRandom<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFillRandom<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFillRandom<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFillRandom<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillRandom<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -990,6 +1076,50 @@ void DeviceAllocation::initialize_sequential_device(Distribution dist) {
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(pointer_),
capacity_,
static_cast<cutlass::float_ue4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_ue4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e2m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::device::BlockFillSequential<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e3m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e3m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::device::BlockFillSequential<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e2m1_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m1_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::device::BlockFillSequential<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(pointer_),
capacity_,
static_cast<cutlass::float_ue8m0_t>(dist.sequential.delta),
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::device::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(pointer_),
@@ -1220,6 +1350,50 @@ void DeviceAllocation::initialize_sequential_host(Distribution dist) {
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_ue4m3_t>(
reinterpret_cast<cutlass::float_ue4m3_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_ue4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_ue4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_e2m3_t>(
reinterpret_cast<cutlass::float_e2m3_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e2m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFillSequential<cutlass::float_e3m2_t>(
reinterpret_cast<cutlass::float_e3m2_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e3m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e3m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFillSequential<cutlass::float_e2m1_t>(
reinterpret_cast<cutlass::float_e2m1_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e2m1_t>(dist.sequential.delta),
static_cast<cutlass::float_e2m1_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFillSequential<cutlass::float_ue8m0_t>(
reinterpret_cast<cutlass::float_ue8m0_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_ue8m0_t>(dist.sequential.delta),
static_cast<cutlass::float_ue8m0_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
@@ -1516,6 +1690,34 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float_e5m2_t const *>(ptr_A),
reinterpret_cast<float_e5m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
reinterpret_cast<float_ue4m3_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFUE8M0:
return reference::device::BlockCompareEqual<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t const *>(ptr_A),
reinterpret_cast<float_ue8m0_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE2M3:
return reference::device::BlockCompareEqual<float_e2m3_t>(
reinterpret_cast<float_e2m3_t const *>(ptr_A),
reinterpret_cast<float_e2m3_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE3M2:
return reference::device::BlockCompareEqual<float_e3m2_t>(
reinterpret_cast<float_e3m2_t const *>(ptr_A),
reinterpret_cast<float_e3m2_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kFE2M1:
return reference::device::BlockCompareEqual<float_e2m1_t>(
reinterpret_cast<float_e2m1_t const *>(ptr_A),
reinterpret_cast<float_e2m1_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF16:
return reference::device::BlockCompareEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -1684,6 +1886,46 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<float_e5m2_t>(epsilon),
static_cast<float_e5m2_t>(nonzero_floor));
case library::NumericTypeID::kFUE4M3:
return reference::device::BlockCompareRelativelyEqual<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t const *>(ptr_A),
reinterpret_cast<float_ue4m3_t const *>(ptr_B),
capacity,
static_cast<float_ue4m3_t>(epsilon),
static_cast<float_ue4m3_t>(nonzero_floor));
case library::NumericTypeID::kFUE8M0:
return reference::device::BlockCompareRelativelyEqual<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t const *>(ptr_A),
reinterpret_cast<float_ue8m0_t const *>(ptr_B),
capacity,
static_cast<float_ue8m0_t>(epsilon),
static_cast<float_ue8m0_t>(nonzero_floor));
case library::NumericTypeID::kFE2M3:
return reference::device::BlockCompareRelativelyEqual<float_e2m3_t>(
reinterpret_cast<float_e2m3_t const *>(ptr_A),
reinterpret_cast<float_e2m3_t const *>(ptr_B),
capacity,
static_cast<float_e2m3_t>(epsilon),
static_cast<float_e2m3_t>(nonzero_floor));
case library::NumericTypeID::kFE3M2:
return reference::device::BlockCompareRelativelyEqual<float_e3m2_t>(
reinterpret_cast<float_e3m2_t const *>(ptr_A),
reinterpret_cast<float_e3m2_t const *>(ptr_B),
capacity,
static_cast<float_e3m2_t>(epsilon),
static_cast<float_e3m2_t>(nonzero_floor));
case library::NumericTypeID::kFE2M1:
return reference::device::BlockCompareRelativelyEqual<float_e2m1_t>(
reinterpret_cast<float_e2m1_t const *>(ptr_A),
reinterpret_cast<float_e2m1_t const *>(ptr_B),
capacity,
static_cast<float_e2m1_t>(epsilon),
static_cast<float_e2m1_t>(nonzero_floor));
case library::NumericTypeID::kF16:
return reference::device::BlockCompareRelativelyEqual<half_t>(
reinterpret_cast<half_t const *>(ptr_A),
@@ -2026,6 +2268,27 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kFE5M2:
write_tensor_csv_static_type<float_e5m2_t>(out, *this);
break;
case library::NumericTypeID::kFUE4M3:
write_tensor_csv_static_type<float_ue4m3_t>(out, *this);
break;
case library::NumericTypeID::kFE2M3:
write_tensor_csv_static_type<float_e2m3_t>(out, *this);
break;
case library::NumericTypeID::kFE3M2:
write_tensor_csv_static_type<float_e3m2_t>(out, *this);
break;
case library::NumericTypeID::kFE2M1:
write_tensor_csv_static_type<float_e2m1_t>(out, *this);
break;
case library::NumericTypeID::kFUE8M0:
write_tensor_csv_static_type<float_ue8m0_t>(out, *this);
break;
case library::NumericTypeID::kF16:
write_tensor_csv_static_type<half_t>(out, *this);
break;
@@ -2193,6 +2456,27 @@ void DeviceAllocation::fill_device(double val = 0.0) {
case library::NumericTypeID::kFE5M2:
tensor_fill<float_e5m2_t>(*this, static_cast<float_e5m2_t>(val));
break;
case library::NumericTypeID::kFUE4M3:
tensor_fill<float_ue4m3_t>(*this, static_cast<float_ue4m3_t>(val));
break;
case library::NumericTypeID::kFUE8M0:
tensor_fill<float_ue8m0_t>(*this, static_cast<float_ue8m0_t>(val));
break;
case library::NumericTypeID::kFE2M3:
tensor_fill<float_e2m3_t>(*this, static_cast<float_e2m3_t>(val));
break;
case library::NumericTypeID::kFE3M2:
tensor_fill<float_e3m2_t>(*this, static_cast<float_e3m2_t>(val));
break;
case library::NumericTypeID::kFE2M1:
tensor_fill<float_e2m1_t>(*this, static_cast<float_e2m1_t>(val));
break;
case library::NumericTypeID::kF16:
tensor_fill<half_t>(*this, static_cast<half_t>(val));
break;
@@ -2288,6 +2572,47 @@ void DeviceAllocation::fill_host(double val = 0.0) {
std::vector<uint8_t> host_data(bytes());
switch (this->type()) {
case library::NumericTypeID::kFUE4M3:
cutlass::reference::host::BlockFill<float_ue4m3_t>(
reinterpret_cast<float_ue4m3_t *>(host_data.data()),
capacity_,
static_cast<float_ue4m3_t>(val)
);
break;
case library::NumericTypeID::kFUE8M0:
cutlass::reference::host::BlockFill<float_ue8m0_t>(
reinterpret_cast<float_ue8m0_t *>(host_data.data()),
capacity_,
static_cast<float_ue8m0_t>(val)
);
break;
case library::NumericTypeID::kFE2M3:
cutlass::reference::host::BlockFill<float_e2m3_t>(
reinterpret_cast<float_e2m3_t *>(host_data.data()),
capacity_,
static_cast<float_e2m3_t>(val)
);
break;
case library::NumericTypeID::kFE3M2:
cutlass::reference::host::BlockFill<float_e3m2_t>(
reinterpret_cast<float_e3m2_t *>(host_data.data()),
capacity_,
static_cast<float_e3m2_t>(val)
);
break;
case library::NumericTypeID::kFE2M1:
cutlass::reference::host::BlockFill<float_e2m1_t>(
reinterpret_cast<float_e2m1_t *>(host_data.data()),
capacity_,
static_cast<float_e2m1_t>(val)
);
break;
case library::NumericTypeID::kFE4M3:
cutlass::reference::host::BlockFill<float_e4m3_t>(
reinterpret_cast<float_e4m3_t *>(host_data.data()),
tools/profiler/src/device_context.cu
+19
View File
@@ -104,6 +104,25 @@ DeviceAllocation *DeviceContext::allocate_and_initialize_tensor(
case library::NumericTypeID::kFE5M2:
data_distribution.set_uniform(-1, 1, 0);
break;
case library::NumericTypeID::kFE2M3:
data_distribution.set_uniform(-2, 2, 0);
break;
case library::NumericTypeID::kFE3M2:
data_distribution.set_uniform(-2, 2, 0);
break;
case library::NumericTypeID::kFE2M1:
data_distribution.set_uniform(-2, 2, 0);
break;
case library::NumericTypeID::kFUE8M0:
data_distribution.set_uniform(1, 4, 0);
break;
case library::NumericTypeID::kFUE4M3:
data_distribution.set_uniform(1, 4, 0);
break;
case library::NumericTypeID::kF16:
data_distribution.set_uniform(-3, 3, 0);
break;
tools/profiler/src/gemm_operation_profiler.cu
+108
View File
@@ -76,6 +76,8 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
{ArgumentTypeID::kEnumerated, {"raster_order", "raster-order"}, "Raster order (heuristic, along_n, along_m)"},
{ArgumentTypeID::kEnumerated, {"runtime_input_datatype_a", "runtime-input-datatype::a"}, "Runtime datatype (e4m3, e5m2, e3m2, e2m3, e2m1)"},
{ArgumentTypeID::kEnumerated, {"runtime_input_datatype_b", "runtime-input-datatype::b"}, "Runtime datatype (e4m3, e5m2, e3m2, e2m3, e2m1)"},
{ArgumentTypeID::kInteger, {"use_pdl", "use-pdl"}, "Use PDL (true, false)"},
{ArgumentTypeID::kInteger, {"swizzle_size", "swizzle-size"}, "Size to swizzle"},
},
@@ -172,6 +174,38 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
}
if (!arg_as_int(this->cluster_n, "cluster_n", problem_space, problem)) {
// default value
this->cluster_n = 1;
}
if (!arg_as_int(this->cluster_k, "cluster_k", problem_space, problem)) {
// default value
this->cluster_k = 1;
}
if (!arg_as_int(this->cluster_m_fallback, "cluster_m_fallback", problem_space, problem)) {
// default value
this->cluster_m_fallback = 0;
}
if (!arg_as_int(this->cluster_n_fallback, "cluster_n_fallback", problem_space, problem)) {
// default value
this->cluster_n_fallback = 0;
}
if (!arg_as_int(this->cluster_k_fallback, "cluster_k_fallback", problem_space, problem)) {
// default value
this->cluster_k_fallback = 0;
}
if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {
// default value
this->use_pdl = false;
@@ -192,6 +226,18 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->split_k_slices = 1;
}
if (!arg_as_RuntimeDatatype(this->runtime_input_datatype_a, "runtime_input_datatype_a", problem_space, problem)) {
// default value
this->runtime_input_datatype_a = cutlass::library::RuntimeDatatype::kStatic;
}
if (!arg_as_RuntimeDatatype(this->runtime_input_datatype_b, "runtime_input_datatype_b", problem_space, problem)) {
// default value
this->runtime_input_datatype_b = cutlass::library::RuntimeDatatype::kStatic;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
@@ -338,6 +384,15 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "cluster_m", problem_space, cluster_m);
set_argument(result, "cluster_n", problem_space, cluster_n);
set_argument(result, "cluster_k", problem_space, cluster_k);
set_argument(result, "cluster_m_fallback", problem_space, cluster_m_fallback);
set_argument(result, "cluster_n_fallback", problem_space, cluster_n_fallback);
set_argument(result, "cluster_k_fallback", problem_space, cluster_k_fallback);
set_argument(result, "split_k_mode", problem_space, library::to_string(split_k_mode));
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
@@ -345,6 +400,11 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "use_pdl", problem_space, library::to_string(use_pdl));
set_argument(result, "runtime_input_datatype_a", problem_space, library::to_string(runtime_input_datatype_a));
set_argument(result, "runtime_input_datatype_b", problem_space, library::to_string(runtime_input_datatype_b));
set_argument(result, "alpha", problem_space,
library::lexical_cast(alpha, operation_desc.element_epilogue));
@@ -388,6 +448,14 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].configuration.problem_size.m() = int(problem_.m);
gemm_workspace_[i].configuration.problem_size.n() = int(problem_.n);
gemm_workspace_[i].configuration.problem_size.k() = int(problem_.k);
gemm_workspace_[i].configuration.cluster_shape.m() = int(problem_.cluster_m);
gemm_workspace_[i].configuration.cluster_shape.n() = int(problem_.cluster_n);
gemm_workspace_[i].configuration.cluster_shape.k() = int(problem_.cluster_k);
gemm_workspace_[i].configuration.cluster_shape_fallback.m() = int(problem_.cluster_m_fallback);
gemm_workspace_[i].configuration.cluster_shape_fallback.n() = int(problem_.cluster_n_fallback);
gemm_workspace_[i].configuration.cluster_shape_fallback.k() = int(problem_.cluster_k_fallback);
gemm_workspace_[i].configuration.lda = problem_.lda;
gemm_workspace_[i].configuration.ldb = problem_.ldb;
gemm_workspace_[i].configuration.ldc = problem_.ldc;
@@ -423,6 +491,15 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_[i].arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_[i].arguments.swizzle_size = problem_.swizzle_size;
gemm_workspace_[i].arguments.raster_order = problem_.raster_order;
gemm_workspace_[i].arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_[i].arguments.runtime_input_datatype_a = problem_.runtime_input_datatype_a;
gemm_workspace_[i].arguments.runtime_input_datatype_b = problem_.runtime_input_datatype_b;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
if (const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments); can_implement != Status::kSuccess) {
return can_implement;
@@ -621,6 +698,9 @@ Status GemmOperationProfiler::initialize_workspace(
if (options.execution_mode != ExecutionMode::kDryRun) {
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_[i].arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_[i].arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
gemm_workspace_[i].arguments.lda = problem_.lda;
gemm_workspace_[i].arguments.ldb = problem_.ldb;
@@ -857,12 +937,32 @@ bool GemmOperationProfiler::verify_cutlass(
}
#endif // #if CUTLASS_ENABLE_CUBLAS
cutlass::library::RuntimeDatatype runtime_datatype_a = gemm_workspace_.front().arguments.runtime_input_datatype_a;
cutlass::library::RuntimeDatatype runtime_datatype_b = gemm_workspace_.front().arguments.runtime_input_datatype_b;
bool is_runtime_datatype_a = runtime_datatype_a != cutlass::library::RuntimeDatatype::kStatic;
bool is_runtime_datatype_b = runtime_datatype_b != cutlass::library::RuntimeDatatype::kStatic;
assert(is_runtime_datatype_a == is_runtime_datatype_b && "runtime datatype should be both dynamic or static.");
library::GemmDescription const &gemm_desc =
static_cast<library::GemmDescription const &>(operation->description());
cutlass::library::NumericTypeID element_A = gemm_desc.A.element;
cutlass::library::NumericTypeID element_B = gemm_desc.B.element;
if (is_runtime_datatype_a) {
element_A = cutlass::library::dynamic_datatype_to_id(runtime_datatype_a);
}
if (is_runtime_datatype_b) {
element_B = cutlass::library::dynamic_datatype_to_id(runtime_datatype_b);
}
bool verification_status = verify_with_reference_(options, report, device_context, operation, problem_space, problem, element_A, element_B);
// Update disposition to worst case verification outcome among all
@@ -1087,6 +1187,14 @@ bool GemmOperationProfiler::verify_with_reference_(
gemm_workspace_[i].configuration.problem_size.m(),
gemm_workspace_[i].configuration.problem_size.n(),
gemm_workspace_[i].configuration.problem_size.k(),
gemm_workspace_[i].configuration.cluster_shape.m(),
gemm_workspace_[i].configuration.cluster_shape.n(),
gemm_workspace_[i].configuration.cluster_shape.k(),
gemm_workspace_[i].configuration.cluster_shape_fallback.m(),
gemm_workspace_[i].configuration.cluster_shape_fallback.n(),
gemm_workspace_[i].configuration.cluster_shape_fallback.k(),
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
tools/profiler/src/operation_profiler.cu
+17
View File
@@ -91,6 +91,11 @@ OperationProfiler::OperationProfiler(
{ArgumentTypeID::kInteger, {"cluster_m", "cluster-shape::m"}, "Cluster shape in the M dimension"},
{ArgumentTypeID::kInteger, {"cluster_n", "cluster-shape::n"}, "Cluster shape in the N dimension"},
{ArgumentTypeID::kInteger, {"cluster_k", "cluster-shape::k"}, "Cluster shape in the K dimension"},
{ArgumentTypeID::kInteger, {"cluster_m_fallback", "cluster-shape-fallback::m"}, "Fallback Cluster shape in the M dimension"},
{ArgumentTypeID::kInteger, {"cluster_n_fallback", "cluster-shape-fallback::n"}, "Fallback Cluster shape in the N dimension"},
{ArgumentTypeID::kInteger, {"cluster_k_fallback", "cluster-shape-fallback::k"}, "Fallback Cluster shape in the K dimension"},
{ArgumentTypeID::kInteger, {"stages", "threadblock-stages"}, "Number of stages of threadblock-scoped matrix multiply"},
{ArgumentTypeID::kInteger, {"warps_m", "warp-count::m"}, "Number of warps within threadblock along the M dimension"},
{ArgumentTypeID::kInteger, {"warps_n", "warp-count::n"}, "Number of warps within threadblock along the N dimension"},
@@ -174,6 +179,11 @@ bool OperationProfiler::satisfies(
return false;
}
}
bool dynamic_cluster = int64_t(op_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(op_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(op_desc.tile_description.cluster_shape.k()) == 0;
int64_t int_value;
if (arg_as_int(int_value, "inst_m", problem_space, problem)) {
@@ -212,6 +222,7 @@ bool OperationProfiler::satisfies(
}
}
if (!dynamic_cluster) {
if (arg_as_int(int_value, "cluster_m", problem_space, problem)) {
if (int64_t(op_desc.tile_description.cluster_shape.m()) != int_value) {
return false;
@@ -230,6 +241,7 @@ bool OperationProfiler::satisfies(
}
}
}
if (arg_as_int(int_value, "stages", problem_space, problem)) {
if (int64_t(op_desc.tile_description.threadblock_stages) != int_value) {
return false;
@@ -296,6 +308,11 @@ std::ostream& operator<<(std::ostream& out, library::OperationKind provider) {
if (provider == library::OperationKind::kGemm) {
out << "kGemm";
}
else if (provider == library::OperationKind::kBlockScaledGemm) {
out << "kBlockScaledGemm";
}
else if (provider == library::OperationKind::kRankK) {
out << "kRankK";
}
tools/profiler/src/options.cu
+21 -4
View File
@@ -33,6 +33,7 @@
*/
#include <algorithm>
#include <fstream>
#include <set>
#include "cutlass/cutlass.h"
@@ -810,16 +811,27 @@ Options::Options(cutlass::CommandLine const &cmdline):
}
else if (cmdline.check_cmd_line_flag("kernels")) {
cmdline.get_cmd_line_arguments("kernels", operation_names);
profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
}
if (cmdline.check_cmd_line_flag("kernels-file")) {
std::string filename;
cmdline.get_cmd_line_argument("kernels-file", filename, {});
std::ifstream input(filename);
if (!input.good()) {
throw std::runtime_error("failed to open: " + filename);
}
for (std::string line; getline(input, line);) {
operation_names.push_back(line);
}
}
if (cmdline.check_cmd_line_flag("ignore-kernels")) {
cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
}
profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
profiling.error_if_nothing_is_profiled = cmdline.check_cmd_line_flag("error-if-nothing-is-profiled");
// Prevent launches on the device for anything other than CUTLASS operation
// Allow verification only on host
if (execution_mode == ExecutionMode::kTrace) {
@@ -856,6 +868,11 @@ void Options::print_usage(std::ostream &out) const {
<< " (\"s1688\" and \"nt\") or (\"s844\" and \"tn\" and \"align8\") in their" << end_of_line
<< " operation name using --kernels=\"s1688*nt, s884*tn*align8\"\n\n"
<< " --kernels-file=<filename> "
<< " Same behavior as --kernels, but kernel names are specified in a file" << end_of_line
<< " with one kernel on each line. Set of profiled kernels is the union of kernels specified" << end_of_line
<< " here and those specified in `kernels`.\n\n"
<< " --ignore-kernels=<string_list> "
<< " Excludes kernels whose names match anything in this list.\n\n"
;
tools/profiler/src/problem_space.cpp
+41
View File
@@ -879,6 +879,32 @@ bool arg_as_NumericTypeID(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_RuntimeDatatype(
library::RuntimeDatatype &runtime_datatype,
KernelArgument::Value const *value_ptr) {
if (value_ptr->not_null) {
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
runtime_datatype = library::from_string<library::RuntimeDatatype>(
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
if (runtime_datatype == library::RuntimeDatatype::kInvalid) {
throw std::runtime_error(
"arg_as_RuntimeDatatype() - illegal cast.");
}
}
else {
throw std::runtime_error(
"arg_as_RuntimeDatatype() - illegal cast.");
}
return true;
}
return false;
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_RasterOrder(
library::RasterOrder &raster_order,
@@ -945,6 +971,21 @@ bool arg_as_LayoutTypeID(
return false;
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_RuntimeDatatype(
library::RuntimeDatatype &runtime_datatype,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
size_t idx = problem_space.argument_index(name);
KernelArgument::Value const *value_ptr = problem.at(idx).get();
return arg_as_RuntimeDatatype(runtime_datatype, value_ptr);
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_LayoutTypeID(
library::LayoutTypeID &layout_type,