leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

CUTLASS 3.2 (#1024)

Browse Source 
* CUTLASS 3.2
This commit is contained in:
ANIKET SHIVAM
2023-08-07 20:50:32 -04:00
committed by GitHub
parent a0d787b746
commit 4575443d44
392 changed files with 47559 additions and 7940 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/cutlass_profiler.cu
-1
View File
@@ -130,7 +130,6 @@ int CutlassProfiler::operator()() {
// Enumerates all operations
enumerate_();
}
return 0;
}
tools/profiler/src/device_allocation.cu
+459
View File
@@ -938,6 +938,465 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
copy_from_host(host_data.data());
}
void DeviceAllocation::initialize_sequential_device(Distribution dist) {
if (!bytes()) {
#ifndef NDEBUG
std::cout << "Skipping initialization of size 0 allocation\n";
#endif
return;
}
if (!data()) {
throw std::runtime_error("Attempting to initialize invalid allocation.");
}
switch (type_) {
case library::NumericTypeID::kFE4M3:
cutlass::reference::device::BlockFillSequential<cutlass::float_e4m3_t>(
reinterpret_cast<cutlass::float_e4m3_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE5M2:
cutlass::reference::device::BlockFillSequential<cutlass::float_e5m2_t>(
reinterpret_cast<cutlass::float_e5m2_t *>(pointer_),
capacity_,
static_cast<cutlass::float_e5m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::device::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(pointer_),
capacity_,
static_cast<cutlass::half_t>(dist.sequential.delta),
static_cast<cutlass::half_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kBF16:
cutlass::reference::device::BlockFillSequential<cutlass::bfloat16_t>(
reinterpret_cast<cutlass::bfloat16_t *>(pointer_),
capacity_,
static_cast<cutlass::bfloat16_t>(dist.sequential.delta),
static_cast<cutlass::bfloat16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kTF32:
cutlass::reference::device::BlockFillSequential<cutlass::tfloat32_t>(
reinterpret_cast<cutlass::tfloat32_t *>(pointer_),
capacity_,
static_cast<cutlass::tfloat32_t>(dist.sequential.delta),
static_cast<cutlass::tfloat32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF32:
cutlass::reference::device::BlockFillSequential<float>(
reinterpret_cast<float *>(pointer_),
capacity_,
static_cast<float>(dist.sequential.delta),
static_cast<float>(dist.sequential.start)
);
break;
case library::NumericTypeID::kCF16:
cutlass::reference::device::BlockFillSequential<cutlass::complex<cutlass::half_t>>(
reinterpret_cast<cutlass::complex<cutlass::half_t> *>(pointer_),
capacity_,
cutlass::complex<cutlass::half_t>(
static_cast<cutlass::half_t>(dist.sequential.delta)),
cutlass::complex<cutlass::half_t>(
static_cast<cutlass::half_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCBF16:
cutlass::reference::device::BlockFillSequential<cutlass::complex<cutlass::bfloat16_t>>(
reinterpret_cast<cutlass::complex<cutlass::bfloat16_t> *>(pointer_),
capacity_,
cutlass::complex<cutlass::bfloat16_t>(
static_cast<cutlass::bfloat16_t>(dist.sequential.delta)),
cutlass::complex<cutlass::bfloat16_t>(
static_cast<cutlass::bfloat16_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCTF32:
cutlass::reference::device::BlockFillSequential<cutlass::complex<cutlass::tfloat32_t>>(
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(pointer_),
capacity_,
cutlass::complex<cutlass::tfloat32_t>(
static_cast<cutlass::tfloat32_t>(dist.sequential.delta)),
cutlass::complex<cutlass::tfloat32_t>(
static_cast<cutlass::tfloat32_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::device::BlockFillSequential<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(pointer_),
capacity_,
cutlass::complex<float>(
static_cast<float>(dist.sequential.delta)),
cutlass::complex<float>(
static_cast<float>(dist.sequential.start))
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::device::BlockFillSequential<double>(
reinterpret_cast<double *>(pointer_),
capacity_,
static_cast<double>(dist.sequential.delta),
static_cast<double>(dist.sequential.start)
);
break;
case library::NumericTypeID::kCF64:
cutlass::reference::device::BlockFillSequential<cutlass::complex<double>>(
reinterpret_cast<cutlass::complex<double> *>(pointer_),
capacity_,
cutlass::complex<double>(
static_cast<double>(dist.sequential.delta)),
cutlass::complex<double>(
static_cast<double>(dist.sequential.start))
);
break;
case library::NumericTypeID::kS2:
cutlass::reference::device::BlockFillSequential<int2b_t>(
reinterpret_cast<int2b_t *>(pointer_),
capacity_,
static_cast<int2b_t>(dist.sequential.delta),
static_cast<int2b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS4:
cutlass::reference::device::BlockFillSequential<int4b_t>(
reinterpret_cast<int4b_t *>(pointer_),
capacity_,
static_cast<int4b_t>(dist.sequential.delta),
static_cast<int4b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::device::BlockFillSequential<int8_t>(
reinterpret_cast<int8_t *>(pointer_),
capacity_,
static_cast<int8_t>(dist.sequential.delta),
static_cast<int8_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS16:
cutlass::reference::device::BlockFillSequential<int16_t>(
reinterpret_cast<int16_t *>(pointer_),
capacity_,
static_cast<int16_t>(dist.sequential.delta),
static_cast<int16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS32:
cutlass::reference::device::BlockFillSequential<int32_t>(
reinterpret_cast<int32_t *>(pointer_),
capacity_,
static_cast<int32_t>(dist.sequential.delta),
static_cast<int32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS64:
cutlass::reference::device::BlockFillSequential<int64_t>(
reinterpret_cast<int64_t *>(pointer_),
capacity_,
static_cast<int64_t>(dist.sequential.delta),
static_cast<int64_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kB1:
cutlass::reference::device::BlockFillSequential<uint1b_t>(
reinterpret_cast<uint1b_t *>(pointer_),
capacity_,
static_cast<uint1b_t>(dist.sequential.delta),
static_cast<uint1b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU2:
cutlass::reference::device::BlockFillSequential<uint2b_t>(
reinterpret_cast<uint2b_t *>(pointer_),
capacity_,
static_cast<uint2b_t>(dist.sequential.delta),
static_cast<uint2b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU4:
cutlass::reference::device::BlockFillSequential<uint4b_t>(
reinterpret_cast<uint4b_t *>(pointer_),
capacity_,
static_cast<uint4b_t>(dist.sequential.delta),
static_cast<uint4b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU8:
cutlass::reference::device::BlockFillSequential<uint8_t>(
reinterpret_cast<uint8_t *>(pointer_),
capacity_,
static_cast<uint8_t>(dist.sequential.delta),
static_cast<uint8_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU16:
cutlass::reference::device::BlockFillSequential<uint16_t>(
reinterpret_cast<uint16_t *>(pointer_),
capacity_,
static_cast<uint16_t>(dist.sequential.delta),
static_cast<uint16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU32:
cutlass::reference::device::BlockFillSequential<uint32_t>(
reinterpret_cast<uint32_t *>(pointer_),
capacity_,
static_cast<uint32_t>(dist.sequential.delta),
static_cast<uint32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU64:
cutlass::reference::device::BlockFillSequential<uint64_t>(
reinterpret_cast<uint64_t *>(pointer_),
capacity_,
static_cast<uint64_t>(dist.sequential.delta),
static_cast<uint64_t>(dist.sequential.start)
);
break;
default: break;
}
}
void DeviceAllocation::initialize_sequential_host(Distribution dist) {
if (!bytes()) {
#ifndef NDEBUG
std::cout << "Skipping initialization of size 0 allocation\n";
#endif
return;
}
if (!data()) {
throw std::runtime_error("Attempting to initialize invalid allocation.");
}
std::vector<uint8_t> host_data(bytes());
switch (type_) {
case library::NumericTypeID::kFE4M3:
cutlass::reference::host::BlockFillSequential<cutlass::float_e4m3_t>(
reinterpret_cast<cutlass::float_e4m3_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e4m3_t>(dist.sequential.delta),
static_cast<cutlass::float_e4m3_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kFE5M2:
cutlass::reference::host::BlockFillSequential<cutlass::float_e5m2_t>(
reinterpret_cast<cutlass::float_e5m2_t *>(host_data.data()),
capacity_,
static_cast<cutlass::float_e5m2_t>(dist.sequential.delta),
static_cast<cutlass::float_e5m2_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF16:
cutlass::reference::host::BlockFillSequential<cutlass::half_t>(
reinterpret_cast<cutlass::half_t *>(host_data.data()),
capacity_,
static_cast<cutlass::half_t>(dist.sequential.delta),
static_cast<cutlass::half_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kBF16:
cutlass::reference::host::BlockFillSequential<cutlass::bfloat16_t>(
reinterpret_cast<cutlass::bfloat16_t *>(host_data.data()),
capacity_,
static_cast<cutlass::bfloat16_t>(dist.sequential.delta),
static_cast<cutlass::bfloat16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kTF32:
cutlass::reference::host::BlockFillSequential<cutlass::tfloat32_t>(
reinterpret_cast<cutlass::tfloat32_t *>(host_data.data()),
capacity_,
static_cast<cutlass::tfloat32_t>(dist.sequential.delta),
static_cast<cutlass::tfloat32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kF32:
cutlass::reference::host::BlockFillSequential<float>(
reinterpret_cast<float *>(host_data.data()),
capacity_,
static_cast<float>(dist.sequential.delta),
static_cast<float>(dist.sequential.start)
);
break;
case library::NumericTypeID::kCF16:
cutlass::reference::host::BlockFillSequential<cutlass::complex<cutlass::half_t>>(
reinterpret_cast<cutlass::complex<cutlass::half_t> *>(host_data.data()),
capacity_,
cutlass::complex<cutlass::half_t>(
static_cast<cutlass::half_t>(dist.sequential.delta)),
cutlass::complex<cutlass::half_t>(
static_cast<cutlass::half_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCBF16:
cutlass::reference::host::BlockFillSequential<cutlass::complex<cutlass::bfloat16_t>>(
reinterpret_cast<cutlass::complex<cutlass::bfloat16_t> *>(host_data.data()),
capacity_,
cutlass::complex<cutlass::bfloat16_t>(
static_cast<cutlass::bfloat16_t>(dist.sequential.delta)),
cutlass::complex<cutlass::bfloat16_t>(
static_cast<cutlass::bfloat16_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCTF32:
cutlass::reference::host::BlockFillSequential<cutlass::complex<cutlass::tfloat32_t>>(
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(host_data.data()),
capacity_,
cutlass::complex<cutlass::tfloat32_t>(
static_cast<cutlass::tfloat32_t>(dist.sequential.delta)),
cutlass::complex<cutlass::tfloat32_t>(
static_cast<cutlass::tfloat32_t>(dist.sequential.start))
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::host::BlockFillSequential<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(host_data.data()),
capacity_,
cutlass::complex<float>(
static_cast<float>(dist.sequential.delta)),
cutlass::complex<float>(
static_cast<float>(dist.sequential.start))
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::host::BlockFillSequential<double>(
reinterpret_cast<double *>(host_data.data()),
capacity_,
static_cast<double>(dist.sequential.delta),
static_cast<double>(dist.sequential.start)
);
break;
case library::NumericTypeID::kCF64:
cutlass::reference::host::BlockFillSequential<cutlass::complex<double>>(
reinterpret_cast<cutlass::complex<double> *>(host_data.data()),
capacity_,
cutlass::complex<double>(
static_cast<double>(dist.sequential.delta)),
cutlass::complex<double>(
static_cast<double>(dist.sequential.start))
);
break;
case library::NumericTypeID::kS2:
cutlass::reference::host::BlockFillSequential<int2b_t>(
reinterpret_cast<int2b_t *>(host_data.data()),
capacity_,
static_cast<int2b_t>(dist.sequential.delta),
static_cast<int2b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS4:
cutlass::reference::host::BlockFillSequential<int4b_t>(
reinterpret_cast<int4b_t *>(host_data.data()),
capacity_,
static_cast<int4b_t>(dist.sequential.delta),
static_cast<int4b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::host::BlockFillSequential<int8_t>(
reinterpret_cast<int8_t *>(host_data.data()),
capacity_,
static_cast<int8_t>(dist.sequential.delta),
static_cast<int8_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS16:
cutlass::reference::host::BlockFillSequential<int16_t>(
reinterpret_cast<int16_t *>(host_data.data()),
capacity_,
static_cast<int16_t>(dist.sequential.delta),
static_cast<int16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS32:
cutlass::reference::host::BlockFillSequential<int32_t>(
reinterpret_cast<int32_t *>(host_data.data()),
capacity_,
static_cast<int32_t>(dist.sequential.delta),
static_cast<int32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kS64:
cutlass::reference::host::BlockFillSequential<int64_t>(
reinterpret_cast<int64_t *>(host_data.data()),
capacity_,
static_cast<int64_t>(dist.sequential.delta),
static_cast<int64_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kB1:
cutlass::reference::host::BlockFillSequential<uint1b_t>(
reinterpret_cast<uint1b_t *>(host_data.data()),
capacity_,
static_cast<uint1b_t>(dist.sequential.delta),
static_cast<uint1b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU2:
cutlass::reference::host::BlockFillSequential<uint2b_t>(
reinterpret_cast<uint2b_t *>(host_data.data()),
capacity_,
static_cast<uint2b_t>(dist.sequential.delta),
static_cast<uint2b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU4:
cutlass::reference::host::BlockFillSequential<uint4b_t>(
reinterpret_cast<uint4b_t *>(host_data.data()),
capacity_,
static_cast<uint4b_t>(dist.sequential.delta),
static_cast<uint4b_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU8:
cutlass::reference::host::BlockFillSequential<uint8_t>(
reinterpret_cast<uint8_t *>(host_data.data()),
capacity_,
static_cast<uint8_t>(dist.sequential.delta),
static_cast<uint8_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU16:
cutlass::reference::host::BlockFillSequential<uint16_t>(
reinterpret_cast<uint16_t *>(host_data.data()),
capacity_,
static_cast<uint16_t>(dist.sequential.delta),
static_cast<uint16_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU32:
cutlass::reference::host::BlockFillSequential<uint32_t>(
reinterpret_cast<uint32_t *>(host_data.data()),
capacity_,
static_cast<uint32_t>(dist.sequential.delta),
static_cast<uint32_t>(dist.sequential.start)
);
break;
case library::NumericTypeID::kU64:
cutlass::reference::host::BlockFillSequential<uint64_t>(
reinterpret_cast<uint64_t *>(host_data.data()),
capacity_,
static_cast<uint64_t>(dist.sequential.delta),
static_cast<uint64_t>(dist.sequential.start)
);
break;
default: break;
}
copy_from_host(host_data.data());
}
void DeviceAllocation::initialize_random_sparsemeta_device(int seed, int MetaSizeInBits) {
if (!bytes()) {
#ifndef NDEBUG
tools/profiler/src/device_allocation.h
+6
View File
@@ -194,6 +194,12 @@ public:
/// Initializes a host allocation to a random distribution using std::cout
void initialize_random_host(int seed, Distribution dist);
/// Initializes a device allocation to a sequential distribution
void initialize_sequential_device(Distribution dist);
/// Initializes a host allocation to a sequential distribution
void initialize_sequential_host(Distribution dist);
/// Initializes a device allocation to a random distribution using cuRAND
void initialize_random_sparsemeta_device(int seed, int MetaSizeInBits);
tools/profiler/src/device_context.cu
+35 -6
View File
@@ -123,15 +123,44 @@ DeviceAllocation *DeviceContext::allocate_tensor(
}
}
// Override pnz for the A/B/C tensors if overridden for Gaussian distributions
if (data_distribution.kind == Distribution::Gaussian) {
double mean = data_distribution.gaussian.mean;
double stddev = data_distribution.gaussian.stddev;
int scale = data_distribution.int_scale;
if (name == "A" && data_distribution.gaussian.pnzA != 100.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzA);
}
else if (name == "B" && data_distribution.gaussian.pnzB != 100.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzB);
}
else if (name == "C" && data_distribution.gaussian.pnzC != 100.0) {
data_distribution.set_gaussian(mean, stddev, scale, data_distribution.gaussian.pnzC);
}
}
if (options.initialization.provider == library::Provider::kReferenceDevice) {
allocation->initialize_random_device(
options.initialization.seed + seed_shift,
data_distribution);
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_device(
data_distribution);
}
else {
allocation->initialize_random_device(
options.initialization.seed + seed_shift,
data_distribution);
}
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
allocation->initialize_random_host(
options.initialization.seed + seed_shift,
data_distribution);
if (data_distribution.kind == Distribution::Sequential) {
allocation->initialize_sequential_host(
data_distribution);
}
else {
allocation->initialize_random_host(
options.initialization.seed + seed_shift,
data_distribution);
}
}
}
tools/profiler/src/gemm_operation_profiler.cu
+28 -19
View File
@@ -322,7 +322,6 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
set_argument(result, "split_k_mode", problem_space, library::to_string(split_k_mode));
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
set_argument(result, "alpha", problem_space,
library::lexical_cast(alpha, operation_desc.element_epilogue));
@@ -377,7 +376,6 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
// initialize reduction operation for parallel splitKMode
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
if (!initialize_reduction_configuration_(operation, problem)) {
@@ -492,7 +490,8 @@ Status GemmOperationProfiler::initialize_workspace(
gemm_workspace_.problem_count = options.profiling.workspace_count;
}
if (options.execution_mode != ExecutionMode::kDryRun) {
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
int seed_shift = 0;
gemm_workspace_.A = device_context.allocate_tensor(
options,
@@ -544,7 +543,9 @@ Status GemmOperationProfiler::initialize_workspace(
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
);
}
if (options.execution_mode != ExecutionMode::kDryRun) {
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_.arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_.arguments.batch_count = problem_.batch_count;
@@ -552,10 +553,10 @@ Status GemmOperationProfiler::initialize_workspace(
gemm_workspace_.arguments.ldb = problem_.ldb;
gemm_workspace_.arguments.ldc = problem_.ldc;
gemm_workspace_.arguments.ldd = problem_.ldc;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
gemm_workspace_.arguments.batch_stride_A = problem_.lda;
gemm_workspace_.arguments.batch_stride_B = problem_.ldb;
gemm_workspace_.arguments.batch_stride_C = problem_.ldc;
gemm_workspace_.arguments.batch_stride_D = problem_.ldc;
/* Query device SM count to pass onto the kernel as an argument, where needed */
gemm_workspace_.arguments.sm_count = options.device.properties.multiProcessorCount;
@@ -739,26 +740,39 @@ bool GemmOperationProfiler::verify_cutlass(
}
#endif // #if CUTLASS_ENABLE_CUBLAS
verify_with_reference_(options, report, device_context, operation, problem_space, problem);
bool verification_status = verify_with_reference_(options, report, device_context, operation, problem_space, problem);
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
for (auto &m : results_.back().verification_map) {
if (m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
results_.back().disposition = m.second;
return true;
}
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
if (!is_any_verification_run_passed && m.second == Disposition::kPassed) {
is_any_verification_run_passed = true;
}
}
if(is_any_verification_run_passed) {
if (is_any_verification_run_passed) {
results_.back().disposition = Disposition::kPassed;
}
}
// if verification.required is set, then return success iff at least one ref-check was run
if (options.verification.required) {
bool did_any_verification_run = false;
for (auto provider : options.verification.providers) {
did_any_verification_run |= (Disposition::kNotRun != results_.back().verification_map[provider]);
}
if (not did_any_verification_run) {
results_.back().status = Status::kErrorNotSupported;
return false;
}
}
// Return true means continue profiling
return true;
}
@@ -902,12 +916,7 @@ bool GemmOperationProfiler::verify_with_reference_(
// Initialize state
//
library::Provider references[] = {
library::Provider::kReferenceDevice,
library::Provider::kReferenceHost
};
for (auto provider : references) {
for (auto provider : options.verification.providers) {
// Skip providers that are not enabled
if (!options.verification.provider_enabled(provider)) {
@@ -994,7 +1003,7 @@ bool GemmOperationProfiler::verify_with_reference_(
if (status != Status::kSuccess) {
results_.back().verification_map[provider] = Disposition::kNotRun;
return true;
continue;
}
results_.back().status = status;
tools/profiler/src/operation_profiler.cu
+2
View File
@@ -386,6 +386,8 @@ int OperationProfiler::profile_all(
operation,
problem_space,
problem);
retval |= (not continue_profiling);
}
if (options.execution_mode == ExecutionMode::kDryRun) {
tools/profiler/src/operation_profiler.h
+3
View File
@@ -39,6 +39,9 @@
#include <memory>
#include <unordered_map>
// CUTLASS includes
#include "cutlass/trace.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
tools/profiler/src/options.cu
+14 -2
View File
@@ -239,11 +239,20 @@ void Options::Initialization::get_distribution(
{"max", &dist.uniform.max},
{"mean", &dist.gaussian.mean},
{"stddev", &dist.gaussian.stddev},
{"pnzA", &dist.gaussian.pnzA},
{"pnzB", &dist.gaussian.pnzB},
{"pnzC", &dist.gaussian.pnzC},
{"start", &dist.sequential.start},
{"delta", &dist.sequential.delta},
{0, 0}
};
// Initalize pnz values to a default value of 100%
dist.gaussian.pnz = 100.0;
dist.gaussian.pnzA = 100.0;
dist.gaussian.pnzB = 100.0;
dist.gaussian.pnzC = 100.0;
using KeyValueVector = std::vector<std::pair<std::string, std::string> >;
KeyValueVector values;
@@ -302,7 +311,7 @@ void Options::Initialization::print_usage(std::ostream &out) const {
<< " --dist=<distribution> "
<< " Data distribution of input tensors {uniform*, gaussian, identity, sequential}" << end_of_line
<< " --dist=uniform,min:<double>,max:<double>,scale:<integer>" << end_of_line
<< " --dist=gaussian,mean:<double>,stddev:<double>,scale:<integer>" << end_of_line
<< " --dist=gaussian,mean:<double>,stddev:<double>,scale:<integer>,pnzA:<double>,pnzB:<double>,pnzC:<double>" << end_of_line
<< " --dist=sequential,start:<double>,delta:<double>,scale:<integer>" << end_of_line
<< " --dist=identity\n\n"
@@ -340,7 +349,7 @@ Options::Library::Library(cutlass::CommandLine const &cmdline) {
for (auto const & token : tokens) {
if (token.find(":")) {
// todo - tokenized range
// TODO: tokenized range
}
else {
int algo;
@@ -473,6 +482,9 @@ size_t Options::Profiling::index(library::Provider provider) const {
Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("verification-enabled", enabled, true);
if (enabled) {
cmdline.get_cmd_line_argument("verification-required", required, false);
}
cmdline.get_cmd_line_argument("epsilon", epsilon, 0.05);
tools/profiler/src/options.h
+4
View File
@@ -150,6 +150,10 @@ public:
/// If true, kernels are verified before they are profiled
bool enabled;
/// If true, causes profiler to return an error code if no reference check is run.
/// Only valid when verification is enabled.
bool required;
/// Relative error threshold - zero to require bit-level consistency
double epsilon;