leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

CUTLASS 2.2 (#96)

Browse Source 
Adds support for NVIDIA Ampere Architecture features. CUDA 11 Toolkit recommended.
This commit is contained in:
Andrew Kerr
2020-06-08 16:17:35 -07:00
committed by GitHub
parent e33d90b361
commit 86931fef85
584 changed files with 51080 additions and 3373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/profiler/src/cublas_helpers.cpp
+131 -5
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -52,15 +52,35 @@ Status get_cutlass_status(cublasStatus_t cublas) {
}
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
cublasOperation_t get_cublas_transpose_operation(library::LayoutTypeID layout) {
bool get_cublas_transpose_operation(
cublasOperation_t &operation,
library::LayoutTypeID layout,
library::ComplexTransform transform) {
switch (layout) {
case library::LayoutTypeID::kColumnMajor:
return CUBLAS_OP_N;
if (transform == library::ComplexTransform::kNone) {
operation = CUBLAS_OP_N;
return true;
}
else {
return false;
}
break;
case library::LayoutTypeID::kRowMajor:
return CUBLAS_OP_T;
if (transform == library::ComplexTransform::kNone) {
operation = CUBLAS_OP_T;
return true;
}
else if (transform == library::ComplexTransform::kConjugate) {
operation = CUBLAS_OP_C;
return true;
}
break;
default: break;
}
throw std::runtime_error("CUTLASS layout type does not correspond to cublas type");
return false;
}
/// Maps a CUTLASS numeric type to a cuBLAS data type enumeration
@@ -114,6 +134,14 @@ bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID ele
case library::NumericTypeID::kB1:
break;
case library::NumericTypeID::kCF32:
data_type = CUDA_C_32F;
return true;
case library::NumericTypeID::kCF64:
data_type = CUDA_C_64F;
return true;
case library::NumericTypeID::kInvalid:
@@ -157,6 +185,104 @@ Status cublas_satisfies(library::GemmDescription const &desc) {
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
cublasGemmExDispatcher::cublasGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmUniversalConfiguration configuration_,
library::GemmUniversalArguments arguments_,
cublasGemmAlgo_t algorithm
):
configuration(configuration_), arguments(arguments_), algo(algorithm), status(Status::kSuccess) {
bool good = true;
good = (good && get_cublas_transpose_operation(trans_A, op_desc.A.layout, op_desc.transform_A));
good = (good && get_cublas_transpose_operation(trans_B, op_desc.B.layout, op_desc.transform_B));
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
good = (good && get_cublas_datatype(
compute_data_type,
op_desc.tile_description.math_instruction.element_accumulator));
// cuBLAS introduces a separate cublasComputeType enumerant to more precisely describe
// internal numerical data types used in the computation.
#if (__CUDA_VER_MAJOR__ >= 11)
library::OpcodeClassID const & opcode_class =
op_desc.tile_description.math_instruction.opcode_class;
if (good &&
op_desc.A.element == library::NumericTypeID::kF32 &&
op_desc.B.element == library::NumericTypeID::kF32 &&
opcode_class == library::OpcodeClassID::kTensorOp) {
compute_type = CUBLAS_COMPUTE_32F_FAST_TF32;
}
else if (good) {
bool const isPedantic = false;
switch (compute_data_type) {
case CUDA_R_32F:
case CUDA_C_32F:
compute_type = isPedantic ? CUBLAS_COMPUTE_32F_PEDANTIC : CUBLAS_COMPUTE_32F;
break;
case CUDA_R_64F:
case CUDA_C_64F:
compute_type = isPedantic ? CUBLAS_COMPUTE_64F_PEDANTIC : CUBLAS_COMPUTE_64F;
break;
case CUDA_R_16F:
compute_type = isPedantic ? CUBLAS_COMPUTE_16F_PEDANTIC : CUBLAS_COMPUTE_16F;
break;
case CUDA_R_32I:
compute_type = isPedantic ? CUBLAS_COMPUTE_32I_PEDANTIC : CUBLAS_COMPUTE_32I;
break;
default:
good = false;
break;
}
}
#endif // __CUDA_VER_MAJOR__ >= 11
if (!good) {
status = Status::kErrorNotSupported;
}
}
/// Executes GEMM using these arguments
cublasStatus_t cublasGemmExDispatcher::operator()(cublasHandle_t handle) {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
#if (__CUDA_VER_MAJOR__ >= 11)
compute_type,
#else
compute_data_type,
#endif
algo
);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
} // namespace profiler
} // namespace cutlass
tools/profiler/src/cublas_helpers.h
+20 -50
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -33,7 +33,10 @@
#include "cutlass/cutlass.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "options.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -45,7 +48,10 @@ namespace profiler {
Status get_cutlass_status(cublasStatus_t cublas);
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
cublasOperation_t get_cublas_transpose_operation(library::LayoutTypeID layout);
bool get_cublas_transpose_operation(
cublasOperation_t &operation,
library::LayoutTypeID layout,
library::ComplexTransform transform = library::ComplexTransform::kNone);
/// Maps a CUTLASS numeric type to a cuBLAS data type enumeration
bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID element_type);
@@ -168,8 +174,8 @@ struct cublasGemmExDispatcher {
//
// Data members
//
library::GemmConfiguration configuration;
library::GemmArguments arguments;
library::GemmUniversalConfiguration configuration;
library::GemmUniversalArguments arguments;
// cublass-specific data structures to fill cublas API call arguments
cublasOperation_t trans_A;
@@ -177,7 +183,12 @@ struct cublasGemmExDispatcher {
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_C;
cudaDataType_t compute_type;
cudaDataType_t compute_data_type;
#if (__CUDA_VER_MAJOR__ >= 11)
cublasComputeType_t compute_type;
#endif
cublasGemmAlgo_t algo;
Status status;
@@ -187,54 +198,13 @@ struct cublasGemmExDispatcher {
cublasGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmConfiguration configuration_,
library::GemmArguments arguments_,
library::GemmUniversalConfiguration configuration_,
library::GemmUniversalArguments arguments_,
cublasGemmAlgo_t algorithm = CUBLAS_GEMM_DFALT
):
configuration(configuration_), arguments(arguments_), algo(algorithm), status(Status::kSuccess) {
trans_A = get_cublas_transpose_operation(op_desc.A.layout);
trans_B = get_cublas_transpose_operation(op_desc.B.layout);
bool good = true;
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
good = (good && get_cublas_datatype(
compute_type,
op_desc.tile_description.math_instruction.element_accumulator));
if (!good) {
status = Status::kErrorNotSupported;
}
}
);
/// Executes GEMM using these arguments
cublasStatus_t operator()(cublasHandle_t handle) {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
compute_type,
algo
);
}
cublasStatus_t operator()(cublasHandle_t handle);
};
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/cutlass_profiler.cu
+3 -10
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -44,7 +44,7 @@ CutlassProfiler::CutlassProfiler(
):
options_(options) {
operation_profilers_.emplace_back(new GemmOperationProfiler);
operation_profilers_.emplace_back(new GemmOperationProfiler(options));
}
@@ -108,13 +108,6 @@ void CutlassProfiler::enumerate_() {
/// Profiles all operations
int CutlassProfiler::profile_() {
library::Manifest manifest(library::Provider::kCUTLASS);
Status status = manifest.initialize();
if (status != Status::kSuccess) {
return -1;
}
int result = 0;
DeviceContext device_context;
@@ -124,7 +117,7 @@ int CutlassProfiler::profile_() {
if (options_.operation_kind == library::OperationKind::kInvalid ||
options_.operation_kind == profiler->kind()) {
result = profiler->profile_all(options_, manifest, device_context);
result = profiler->profile_all(options_, library::Singleton::get().manifest, device_context);
if (result) {
return result;
tools/profiler/src/cutlass_profiler.h
+2 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -30,6 +30,7 @@
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/manifest.h"
#include "cutlass/library/singleton.h"
#include "options.h"
#include "operation_profiler.h"
tools/profiler/src/debug.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/device_allocation.cu
+48 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -431,6 +431,14 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::device::BlockFillRandom<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::device::BlockFillRandom<double>(
reinterpret_cast<double *>(pointer_),
@@ -548,6 +556,14 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::host::BlockFillRandom<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::host::BlockFillRandom<double>(
reinterpret_cast<double *>(host_data.data()),
@@ -655,6 +671,12 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF32:
return reference::device::BlockCompareEqual<cutlass::complex<float> >(
reinterpret_cast<complex<float> const *>(ptr_A),
reinterpret_cast<complex<float> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF16:
return reference::device::BlockCompareEqual<complex<half_t>>(
@@ -825,6 +847,23 @@ bool DeviceAllocation::block_compare_relatively_equal(
static_cast<uint64_t>(epsilon),
static_cast<uint64_t>(nonzero_floor));
// No relatively equal comparison for complex numbers.
//
// As a simplification, we can require bitwise equality. This avoids false positives.
// (i.e. "pass" really means passing. "Fail" may not actually mean failure given appropriate epsilon.)
//
case library::NumericTypeID::kCF32:
return reference::device::BlockCompareEqual<cutlass::complex<float> >(
reinterpret_cast<complex<float> const *>(ptr_A),
reinterpret_cast<complex<float> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF64:
return reference::device::BlockCompareEqual<cutlass::complex<double> >(
reinterpret_cast<complex<double> const *>(ptr_A),
reinterpret_cast<complex<double> const *>(ptr_B),
capacity);
default:
throw std::runtime_error("Unsupported numeric type");
}
@@ -970,6 +1009,14 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kU64:
write_tensor_csv_static_type<uint64_t>(out, *this);
break;
case library::NumericTypeID::kCF32:
write_tensor_csv_static_type<cutlass::complex<float> >(out, *this);
break;
case library::NumericTypeID::kCF64:
write_tensor_csv_static_type<cutlass::complex<double> >(out, *this);
break;
default:
throw std::runtime_error("Unsupported numeric type");
tools/profiler/src/device_allocation.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/device_context.cu
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/device_context.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/enumerated_types.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/enumerated_types.h
+2 -2
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -50,7 +50,7 @@ T from_string(std::string const &);
enum class ExecutionMode {
kProfile, ///< regular verification and profiling
kDryRun, ///< no kernels are launched or workspaces allocated; used to assess what operators might be launched
kEnumerate, ///< no kernels launched or workspaces allocated; lists all function types and functions
kEnumerate, ///< no kernels launched or workspaces allocated; lists all operation kind and operations
kTrace, ///< executes a single device-side computation with no other kernel launches
kInvalid
};
tools/profiler/src/gemm_operation_profiler.cu
+127 -83
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -31,6 +31,8 @@
#include <iomanip>
#include <ios>
#include "cutlass/core_io.h"
#include "cublas_helpers.h"
#include "gemm_operation_profiler.h"
#include "gpu_timer.h"
@@ -44,22 +46,27 @@ namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
GemmOperationProfiler::GemmOperationProfiler():
OperationProfiler(library::OperationKind::kGemm,{
{ArgumentTypeID::kEnumerated, {"Gemm_kind"}, "Variant of GEMM (e.g. gemm, planar complex, batched, ...)"},
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
{ArgumentTypeID::kInteger, {"split_k_slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count"}, "Number of GEMMs computed in one batch"},
}) {
GemmOperationProfiler::GemmOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kGemm,
{
{ArgumentTypeID::kEnumerated, {"gemm_kind"}, "Variant of GEMM (gemm, batched, array, universal, planar_complex, planar_complex_array)"},
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
{ArgumentTypeID::kInteger, {"split_k_slices", "split-k-slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count", "batch-count"}, "Number of GEMMs computed in one batch"},
},
{ library::Provider::kCUBLAS}
) {
description_ = "General matrix-matrix product. D = alpha * A*B + beta * C";
description_ = " General matrix-matrix product. D = alpha * A*B + beta * C";
}
/// Destructor
@@ -107,6 +114,8 @@ void GemmOperationProfiler::print_examples(std::ostream &out) const {
<< " --providers=cutlass --output=functional-test.csv\n\n";
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#if 0
// used this for debugging
static std::string byte_string(std::vector<uint8_t> const &bytes) {
@@ -122,47 +131,34 @@ static std::string byte_string(std::vector<uint8_t> const &bytes) {
}
#endif
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Extracts the problem dimensions
Status GemmOperationProfiler::initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
Status GemmOperationProfiler::GemmProblem::parse(
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
if (operation_desc.gemm_kind != library::GemmKind::kGemm) {
return Status::kErrorInvalidProblem;
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
this->m = 1024;
}
if (!arg_as_int(problem_.m, "m", problem_space, problem)) {
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
problem_.m = 1024;
}
if (!arg_as_int(problem_.n, "n", problem_space, problem)) {
// default value
problem_.n = 1024;
this->n = 1024;
}
if (!arg_as_int(problem_.k, "k", problem_space, problem)) {
if (!arg_as_int(this->k, "k", problem_space, problem)) {
// default value
problem_.k = 1024;
this->k = 1024;
}
if (!arg_as_int(problem_.split_k_slices, "split_k_slices", problem_space, problem)) {
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
problem_.split_k_slices = 1;
this->split_k_slices = 1;
}
if (!arg_as_int(problem_.batch_count, "batch_count", problem_space, problem)) {
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
problem_.batch_count = 1;
this->batch_count = 1;
}
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
@@ -178,37 +174,97 @@ Status GemmOperationProfiler::initialize_configuration(
}
if (!arg_as_scalar(
problem_.alpha,
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(problem_.alpha, operation_desc.element_epilogue, 1)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
problem_.beta,
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(problem_.beta, operation_desc.element_epilogue, 0)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
problem_.lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(problem_.m), int(problem_.k)}).front();
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(this->m), int(this->k)}).front();
problem_.ldb = DeviceAllocation::get_packed_layout(
operation_desc.B.layout, {int(problem_.k), int(problem_.n)}).front();
this->ldb = DeviceAllocation::get_packed_layout(
operation_desc.B.layout, {int(this->k), int(this->n)}).front();
problem_.ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(problem_.m), int(problem_.n)}).front();
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
return Status::kSuccess;
}
/// Initializes a performance result
void GemmOperationProfiler::GemmProblem::initialize_result(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space) {
result.arguments.resize(problem_space.rank());
set_argument(result, "gemm_kind", problem_space, library::to_string(operation_desc.gemm_kind));
set_argument(result, "A", problem_space,
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
set_argument(result, "B", problem_space,
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
set_argument(result, "C", problem_space,
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
set_argument(result, "m", problem_space, m);
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
set_argument(result, "alpha", problem_space,
library::lexical_cast(alpha, operation_desc.element_epilogue));
set_argument(result, "beta", problem_space,
library::lexical_cast(beta, operation_desc.element_epilogue));
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Extracts the problem dimensions
Status GemmOperationProfiler::initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
if (operation_desc.gemm_kind != library::GemmKind::kUniversal) {
return Status::kErrorInvalidProblem;
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
gemm_workspace_.configuration.problem_size.m() = int(problem_.m);
gemm_workspace_.configuration.problem_size.n() = int(problem_.n);
@@ -217,7 +273,8 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_.configuration.ldb = problem_.ldb;
gemm_workspace_.configuration.ldc = problem_.ldc;
gemm_workspace_.configuration.ldd = problem_.ldc;
gemm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
//gemm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
gemm_workspace_.configuration.batch_count = int(problem_.split_k_slices);
gemm_workspace_.arguments.A = nullptr;
gemm_workspace_.arguments.B = nullptr;
@@ -243,37 +300,24 @@ void GemmOperationProfiler::initialize_result_(
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
result.arguments.resize(problem_space.rank());
set_argument_(result, "A", problem_space,
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
set_argument_(result, "B", problem_space,
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
set_argument_(result, "C", problem_space,
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
set_argument_(result, "m", problem_space, problem_.m);
set_argument_(result, "n", problem_space, problem_.n);
set_argument_(result, "k", problem_space, problem_.k);
set_argument_(result, "split_k_slices", problem_space, problem_.split_k_slices);
set_argument_(result, "batch_count", problem_space, problem_.batch_count);
set_argument_(result, "alpha", problem_space,
library::lexical_cast(problem_.alpha, operation_desc.element_epilogue));
set_argument_(result, "beta", problem_space,
library::lexical_cast(problem_.beta, operation_desc.element_epilogue));
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
// Input bytes read and Output bytes written for the gemm problem
result.bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) * problem_.k +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.n / 8) * problem_.k +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n * 2;
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(problem_.beta.begin(), problem_.beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
result.bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
}
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
result.runtime = 0;
@@ -378,8 +422,9 @@ Status GemmOperationProfiler::initialize_workspace(
results_.back().provider = library::Provider::kCUTLASS;
results_.back().op_kind = library::OperationKind::kGemm;
results_.back().disposition = Disposition::kNotRun;
for(auto &verification_provider : options.verification.providers) {
results_.back().verification_map[verification_provider] = Disposition::kNotRun;
for(auto provider : verification_providers_) {
results_.back().verification_map[provider] = Disposition::kNotRun;
}
}
@@ -559,8 +604,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
);
if (gemm_op.status != Status::kSuccess) {
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotRun;
return true;
}
tools/profiler/src/gemm_operation_profiler.h
+17 -4
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -31,6 +31,7 @@
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
@@ -75,6 +76,18 @@ public:
GemmProblem():
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
@@ -86,8 +99,8 @@ public:
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::GemmConfiguration configuration;
library::GemmArguments arguments;
library::GemmUniversalConfiguration configuration;
library::GemmUniversalArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
@@ -122,7 +135,7 @@ public:
//
/// Ctor
GemmOperationProfiler();
GemmOperationProfiler(Options const &options);
/// Destructor
virtual ~GemmOperationProfiler();
tools/profiler/src/gpu_timer.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/gpu_timer.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/main.cpp
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/operation_profiler.cu
+82 -37
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -31,6 +31,7 @@
#include <iomanip>
#include <cstring>
#include <fstream>
#include <sstream>
#ifdef __unix__
#include <unistd.h>
@@ -55,30 +56,41 @@ OperationProfiler::OperationProfiler(): kind_(library::OperationKind::kInvalid)
/// Ctor
OperationProfiler::OperationProfiler(
Options const &options,
library::OperationKind kind,
ArgumentDescriptionVector const &arguments,
ProviderVector const & reference_providers
ProviderVector const & verification_providers
):
kind_(kind), arguments_(arguments), reference_providers_(reference_providers) {
kind_(kind), arguments_(arguments) {
ArgumentDescriptionVector tile_description_arguments{
{ArgumentTypeID::kEnumerated, {"op_class", "opcode-class"}, "Class of math instruction (SIMT or TensorOp)."},
{ArgumentTypeID::kEnumerated, {"accum", "accumulator-type"}, "Math instruction accumulator data type."},
{ArgumentTypeID::kInteger, {"cta_m", "threadblock-shape::m"}, "Threadblock shape in the M dimension."},
{ArgumentTypeID::kInteger, {"cta_n", "threadblock-shape::n"}, "Threadblock shape in the N dimension."},
{ArgumentTypeID::kInteger, {"cta_k", "threadblock-shape::k"}, "Threadblock shape in the K dimension."},
{ArgumentTypeID::kInteger, {"stages", "threadblock-stages"}, "Number of stages of threadblock-scoped matrix multiply."},
{ArgumentTypeID::kInteger, {"warps_m", "warp-count::m"}, "Number of warps within threadblock along the M dimension."},
{ArgumentTypeID::kInteger, {"warps_n", "warp-count::n"}, "Number of warps within threadblock along the N dimension."},
{ArgumentTypeID::kInteger, {"warps_k", "warp-count::k"}, "Number of warps within threadblock along the K dimension."},
{ArgumentTypeID::kInteger, {"inst_m", "instruction-shape::m"}, "Math instruction shape in the M dimension."},
{ArgumentTypeID::kInteger, {"inst_n", "instruction-shape::n"}, "Math instruction shape in the N dimension."},
{ArgumentTypeID::kInteger, {"inst_k", "instruction-shape::k"}, "Math instruction shape in the K dimension."},
{ArgumentTypeID::kInteger, {"min_cc", "minimum-compute-capability"}, "Minimum device compute capability."},
{ArgumentTypeID::kInteger, {"max_cc", "maximum-compute-capability"}, "Maximum device compute capability."}
{ArgumentTypeID::kEnumerated, {"op_class", "opcode-class"}, "Class of math instruction (simt, tensorop, wmmatensorop, wmma)"},
{ArgumentTypeID::kEnumerated, {"accum", "accumulator-type"}, "Math instruction accumulator data type"},
{ArgumentTypeID::kInteger, {"cta_m", "threadblock-shape::m"}, "Threadblock shape in the M dimension"},
{ArgumentTypeID::kInteger, {"cta_n", "threadblock-shape::n"}, "Threadblock shape in the N dimension"},
{ArgumentTypeID::kInteger, {"cta_k", "threadblock-shape::k"}, "Threadblock shape in the K dimension"},
{ArgumentTypeID::kInteger, {"stages", "threadblock-stages"}, "Number of stages of threadblock-scoped matrix multiply"},
{ArgumentTypeID::kInteger, {"warps_m", "warp-count::m"}, "Number of warps within threadblock along the M dimension"},
{ArgumentTypeID::kInteger, {"warps_n", "warp-count::n"}, "Number of warps within threadblock along the N dimension"},
{ArgumentTypeID::kInteger, {"warps_k", "warp-count::k"}, "Number of warps within threadblock along the K dimension"},
{ArgumentTypeID::kInteger, {"inst_m", "instruction-shape::m"}, "Math instruction shape in the M dimension"},
{ArgumentTypeID::kInteger, {"inst_n", "instruction-shape::n"}, "Math instruction shape in the N dimension"},
{ArgumentTypeID::kInteger, {"inst_k", "instruction-shape::k"}, "Math instruction shape in the K dimension"},
{ArgumentTypeID::kInteger, {"min_cc", "minimum-compute-capability"}, "Minimum device compute capability"},
{ArgumentTypeID::kInteger, {"max_cc", "maximum-compute-capability"}, "Maximum device compute capability"}
};
arguments_.insert(arguments_.end(), tile_description_arguments.begin(), tile_description_arguments.end());
for (auto provider : verification_providers) {
if (std::find(
options.verification.providers.begin(),
options.verification.providers.end(),
provider) != options.verification.providers.end()) {
verification_providers_.push_back(provider);
}
}
}
/// Destructor
@@ -248,8 +260,9 @@ int OperationProfiler::profile_all(
auto min_cc = operation->description().tile_description.minimum_compute_capability;
auto max_cc = operation->description().tile_description.maximum_compute_capability;
// Execute compatible operations if they satisfy the current device's compute capability
// Execute compatible cutlass operations if they satisfy the current device's compute capability
if (operation->description().kind == kind_ &&
operation->description().provider == library::Provider::kCUTLASS &&
options.device.compute_capability() >= min_cc &&
options.device.compute_capability() <= max_cc) {
@@ -259,7 +272,7 @@ int OperationProfiler::profile_all(
if (!filtered_by_name) {
for (auto const & op_name : options.operation_names) {
if (operation_name.find(op_name) !=std::string::npos) {
if (find_string_matches_(op_name, operation_name)) {
filtered_by_name = true;
break;
}
@@ -278,7 +291,7 @@ int OperationProfiler::profile_all(
operation,
problem_space,
problem);
if (status == Status::kErrorInternal) {
// Stop profiling if there was an internal error
return false;
@@ -548,29 +561,28 @@ void OperationProfiler::initialize_result_(
library::OperationDescription const &operation_desc,
ProblemSpace const &problem_space) {
set_argument_(result, "op_class", problem_space,
set_argument(result, "op_class", problem_space,
library::to_string(operation_desc.tile_description.math_instruction.opcode_class));
set_argument_(result, "accum", problem_space,
set_argument(result, "accum", problem_space,
library::to_string(operation_desc.tile_description.math_instruction.element_accumulator));
set_argument_(result, "cta_m", problem_space, operation_desc.tile_description.threadblock_shape.m());
set_argument_(result, "cta_n", problem_space, operation_desc.tile_description.threadblock_shape.n());
set_argument_(result, "cta_k", problem_space, operation_desc.tile_description.threadblock_shape.k());
set_argument_(result, "stages", problem_space, operation_desc.tile_description.threadblock_stages);
set_argument_(result, "warps_m", problem_space, operation_desc.tile_description.warp_count.m());
set_argument_(result, "warps_n", problem_space, operation_desc.tile_description.warp_count.n());
set_argument_(result, "warps_k", problem_space, operation_desc.tile_description.warp_count.k());
set_argument_(result, "inst_m", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.m());
set_argument_(result, "inst_n", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.n());
set_argument_(result, "inst_k", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.k());
set_argument_(result, "min_cc", problem_space, operation_desc.tile_description.minimum_compute_capability);
set_argument_(result, "max_cc", problem_space, operation_desc.tile_description.maximum_compute_capability);
set_argument(result, "cta_m", problem_space, operation_desc.tile_description.threadblock_shape.m());
set_argument(result, "cta_n", problem_space, operation_desc.tile_description.threadblock_shape.n());
set_argument(result, "cta_k", problem_space, operation_desc.tile_description.threadblock_shape.k());
set_argument(result, "stages", problem_space, operation_desc.tile_description.threadblock_stages);
set_argument(result, "warps_m", problem_space, operation_desc.tile_description.warp_count.m());
set_argument(result, "warps_n", problem_space, operation_desc.tile_description.warp_count.n());
set_argument(result, "warps_k", problem_space, operation_desc.tile_description.warp_count.k());
set_argument(result, "inst_m", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.m());
set_argument(result, "inst_n", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.n());
set_argument(result, "inst_k", problem_space, operation_desc.tile_description.math_instruction.instruction_shape.k());
set_argument(result, "min_cc", problem_space, operation_desc.tile_description.minimum_compute_capability);
set_argument(result, "max_cc", problem_space, operation_desc.tile_description.maximum_compute_capability);
}
/// Helper
void OperationProfiler::set_argument_(
void OperationProfiler::set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
@@ -579,7 +591,7 @@ void OperationProfiler::set_argument_(
result.arguments.at(problem_space.argument_index(name)) = make_pair(std::string(name), value);
}
void OperationProfiler::set_argument_(
void OperationProfiler::set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
@@ -588,6 +600,39 @@ void OperationProfiler::set_argument_(
result.arguments.at(problem_space.argument_index(name)) = make_pair(std::string(name), library::lexical_cast(value));
}
/// finds string matches filter_string in operation_name
bool OperationProfiler::find_string_matches_(
std::string const &filter_string,
std::string const &operation_name) {
// Returns true if all substrings appear in the operation_name in order
// Split filter_string of the format "gemm*f32*nt" to tokens ["gemm", "f32", "nt"]
std::string item;
std::istringstream iss(filter_string);
std::vector<std::string> filter_tokens;
while (std::getline(iss, item, '*')) {
filter_tokens.push_back(item);
}
// Search filter_tokens in operation_name in order
size_t start = 0, idx = 0;
for(auto & token : filter_tokens) {
// Check if characters left to be parsed in operation_name
if (start < operation_name.length()) {
// Find token in operation_name[start:]
idx = operation_name.substr(start).find(token);
if (idx == std::string::npos) {
return false;
}
}
start += (idx + token.length());
}
// All tokens in filter_string found in operation_name
return true;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
tools/profiler/src/operation_profiler.h
+24 -17
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -73,7 +73,7 @@ protected:
ArgumentDescriptionVector arguments_;
/// List of providers used to verify and compare each result
ProviderVector reference_providers_;
ProviderVector verification_providers_;
/// Model performance result initailized by the operation profiler with workload statistics
/// and reasonable default state.
@@ -92,9 +92,10 @@ public:
OperationProfiler();
OperationProfiler(
Options const &options,
library::OperationKind kind,
ArgumentDescriptionVector const &arguments = ArgumentDescriptionVector(),
ProviderVector const & reference_providers = ProviderVector());
ProviderVector const & verification_providers = ProviderVector());
/// Destructor
virtual ~OperationProfiler();
@@ -196,6 +197,20 @@ public:
library::OperationDescription const &desc,
library::Provider provider,
library::Provider verification_provider = library::Provider::kInvalid);
/// Helper to set a performance result member
static void set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
std::string const &value);
/// Helper to set a performance result member
static void set_argument(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
int64_t value);
protected:
@@ -205,20 +220,6 @@ protected:
library::OperationDescription const &operation_desc,
ProblemSpace const &problem_space);
/// Helper to set a performance result member
static void set_argument_(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
std::string const &value);
/// Helper to set a performance result member
static void set_argument_(
PerformanceResult &result,
char const *name,
ProblemSpace const &problem_space,
int64_t value);
/// Method to profile an initialized CUTLASS operation
virtual Status profile_cutlass_(
double &runtime,
@@ -227,6 +228,12 @@ protected:
void const *arguments,
void *host_workspace,
void *device_workspace);
private:
/// finds string matches filter_string in operation_name
bool find_string_matches_(
std::string const &filter_string,
std::string const &operation_name);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/options.cu
+69 -55
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -76,7 +76,7 @@ Options::Device::Device(cutlass::CommandLine const &cmdline) {
void Options::Device::print_usage(std::ostream &out) const {
out << "Device:\n"
<< " --device=<int> "
<< " --device=<int> "
<< " CUDA Device ID\n\n";
int device_count = 0;
@@ -106,7 +106,7 @@ void Options::Device::print_usage(std::ostream &out) const {
}
out
<< " --compute-capability=<int> "
<< " --compute-capability=<int> "
<< " Override the compute capability.\n\n";
}
@@ -255,12 +255,6 @@ void Options::Initialization::get_distribution(
continue; // next token
}
// Casts as integer without scaling
if (it->first.compare("integer") == 0) {
dist.int_scale = 0;
continue; // next token
}
// initialize other members
for (int m = 0; members[m].label; ++m) {
if (it->first == members[m].label && !it->second.empty()) {
@@ -276,19 +270,23 @@ void Options::Initialization::print_usage(std::ostream &out) const {
out << "Initialization:\n"
<< " --initialization=<bool> "
<< " --initialization=<bool> "
<< " Enables initialization (default: true). If false, device memory is" << end_of_line
<< "not initialized after allocation.\n\n"
<< " not initialized after allocation.\n\n"
<< " --initialization-provider=<provider> "
<< " Selects 'device' or 'host' initialization.\n\n"
<< " --initialization-provider=<provider> "
<< " Selects initialization provider {host, device*}. (default: '*')\n\n"
<< " --dist=<distribution> "
<< " Data distribution of input tensors\n\n"
<< " --dist=<distribution> "
<< " Data distribution of input tensors {uniform*, gaussian, identity, sequential}" << end_of_line
<< " --dist=uniform,min:<double>,max:<double>,scale:<integer>" << end_of_line
<< " --dist=gaussian,mean:<double>,stddev:<double>,scale:<integer>" << end_of_line
<< " --dist=sequential,start:<double>,delta:<double>,scale:<integer>" << end_of_line
<< " --dist=identity\n\n"
<< " --seed=<int> "
<< " --seed=<int> "
<< " Random number generator seed. Used to enforce deterministic" << end_of_line
<< "initialization.\n\n";
<< " initialization.\n\n";
}
@@ -339,12 +337,12 @@ void Options::Library::print_usage(std::ostream &out) const {
out << "Library:\n"
<< " --library-algo-mode=<mode> "
<< " --library-algo-mode=<mode> "
<< " Indicates algorithm mode used to call libraries such as cuBLAS and cuDNN.\n"
<< " "
<< " "
<< " mode={default*,matching,best}\n\n"
<< " --library-algos=<range-list> "
<< " --library-algos=<range-list> "
<< " If --algorithm-mode=best, permits specifying a selection of algorithms.\n\n";
}
@@ -393,21 +391,25 @@ void Options::Profiling::print_usage(std::ostream &out) const {
out << "Profiling:\n"
<< " --profiling-iterations=<iterations> "
<< " --profiling-iterations=<iterations> "
<< " Number of iterations to profile each kernel. If zero, kernels" << end_of_line
<< "are launched up to the profiling duration.\n\n"
<< " are launched up to the profiling duration.\n\n"
<< " --warmup-iterations=<iterations> "
<< " --warmup-iterations=<iterations> "
<< " Number of iterations to execute each kernel prior to profiling.\n\n"
<< " --sleep-duration=<duration> "
<< " Number of ms to sleep between profiling periods (ms)\n\n"
<< " --sleep-duration=<duration> "
<< " Number of ms to sleep between profiling periods (ms).\n\n"
<< " --profiling-enabled=<bool> "
<< " --profiling-enabled=<bool> "
<< " If true, profiling is actually conducted.\n\n"
<< " --providers=<providers> "
<< " List of providers to be profiled for performance\n\n";
<< " --providers=<providers> "
<< " List of providers to be profiled for performance. (default: '*')" << end_of_line
<< " Gemm providers {cutlass*"
<< "}" << end_of_line
<< "\n\n";
}
void Options::Profiling::print_options(std::ostream &out, int indent) const {
@@ -477,6 +479,7 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
}
else {
providers.push_back(library::Provider::kCUBLAS);
providers.push_back(library::Provider::kReferenceDevice);
}
}
@@ -484,22 +487,27 @@ void Options::Verification::print_usage(std::ostream &out) const {
out << "Verification:\n"
<< " --verification-enabled=<bool> "
<< " --verification-enabled=<bool> "
<< " Whether to perform verification checks.\n\n"
<< " --epsilon=<error> "
<< " --epsilon=<error> "
<< " Error threshold. Setting to zero (default) requires" << end_of_line
<< "bit-level equivalence.\n\n"
<< " bit-level equivalence.\n\n"
<< " --nonzero-floor=<floor> "
<< " --nonzero-floor=<floor> "
<< " Results whose absolute value is less than this quantity" << end_of_line
<< "are treated as zero for comparisons.\n\n"
<< " are treated as zero for comparisons.\n\n"
<< " --save-workspace={*never,incorrect,always}"
<< " Specifies when to save the GEMM inputs and results to the filesystem.\n\n"
<< " --save-workspace=<string> "
<< " Specifies when to save the GEMM inputs and results to the filesystem." << end_of_line
<< " --save-workspace=never never save workspace (default)" << end_of_line
<< " --save-workspace=incorrect save workspace for incorrect results" << end_of_line
<< " --save-workspace=always always save workspace\n\n"
<< " --verification-providers=<providers> "
<< " List of providers used to verify result. (default: device)\n\n";
<< " --verification-providers=<providers> "
<< " List of providers used to verify result. (default: '*')" << end_of_line
<< " Gemm verification-providers {cublas*}" << end_of_line
<< "\n\n";
}
void Options::Verification::print_options(std::ostream &out, int indent) const {
@@ -554,22 +562,22 @@ void Options::Report::print_usage(std::ostream &out) const {
out << "Report:\n"
<< " --append=<bool> "
<< " --append=<bool> "
<< " If true, result is appended to possibly existing file. Otherwise, " << end_of_line
<< "any existing file is overwritten.\n\n"
<< " any existing file is overwritten.\n\n"
<< " --output=<path> "
<< " Path to output file for machine readable results.\n\n"
<< " --output=<path> "
<< " Path to output file for machine readable results. Operation kind and '.csv' is appended.\n\n"
<< " --report-not-run=<bool> "
<< " --report-not-run=<bool> "
<< " If true, reports the status of all kernels including those that" << end_of_line
<< "do not satisfy the given arguments.\n\n"
<< " do not satisfy the given arguments.\n\n"
<< " --tags=<column:tag,...> "
<< " --tags=<column:tag,...> "
<< " Inserts leading columns in output table and uniform values for each" << end_of_line
<< "column. Useful for generating pivot tables.\n\n"
<< " column. Useful for generating pivot tables.\n\n"
<< " --verbose=<bool> "
<< " --verbose=<bool> "
<< " Prints human-readable text to stdout. If false, nothing is written to stdout.\n\n";
}
@@ -600,7 +608,7 @@ Options::About::About(cutlass::CommandLine const &cmdline) {
void Options::About::print_usage(std::ostream &out) const {
out << "About:\n"
<< " --version ";
<< " --version ";
print_version(out);
@@ -675,22 +683,29 @@ Options::Options(cutlass::CommandLine const &cmdline):
void Options::print_usage(std::ostream &out) const {
out
<< "CUTLASS Performance Tool\n"
<< "CUTLASS Profiler\n"
<< "usage:\n\n"
<< " cutlass_profiler [options]\n\n"
<< " --help\n\n"
<< " --mode={profile*,single,dry,trace,enumerate} "
<< " Regular profiling, single kernel mode only, or no profiling.\n\n"
<< " --mode=<string> "
<< " Cutlass profiler execution mode." << end_of_line
<< " --mode=profile regular verification and profiling (default)" << end_of_line
<< " --mode=dry_run no kernels are launched or workspaces allocated" << end_of_line
<< " --mode=enumerate lists all operation kind and operations" << end_of_line
<< " --mode=trace executes a single device-side computation with" << end_of_line
<< " no other kernel launches\n\n"
<< " --device-info "
<< " --device-info "
<< " Prints information on all GPUs present in the system\n\n"
<< " --operation=<operation_kind> "
<< " --operation=<operation_kind> "
<< " CUTLASS operation to profile.\n\n"
<< " --kernels=<string_list> "
<< " List of substrings to filter operations by name.\n\n"
<< " --kernels=<string_list> "
<< " Filter operations by kernel names. For example, call all kernels with" << end_of_line
<< " (\"s1688\" and \"nt\") or (\"s844\" and \"tn\" and \"align8\") in their" << end_of_line
<< " operation name using --kernels=\"s1688*nt, s884*tn*align8\"\n\n"
;
//
@@ -755,4 +770,3 @@ std::string Options::indent_str(int indent) {
} // namespace profiler
} // namespace cutlass
tools/profiler/src/options.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/performance_report.cpp
+24 -19
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -68,9 +68,11 @@ PerformanceReport::PerformanceReport(
):
options_(options), argument_names_(argument_names), problem_index_(0), good_(true), op_kind_(op_kind) {
std::string file_name = options_.report.output_path.substr(0, options_.report.output_path.rfind("."));
std::string file_extension = options_.report.output_path.substr(options_.report.output_path.rfind(".") + 1);
op_file_name_ = file_name + "." + to_string(op_kind_) + "." + file_extension;
// Strip '.csv' if present
std::string base_path = options_.report.output_path.substr(
0, options_.report.output_path.rfind(".csv"));
op_file_name_ = base_path + "." + to_string(op_kind_) + ".csv";
//
// Open output file for operation of PerformanceReport::op_kind
@@ -166,6 +168,7 @@ void PerformanceReport::close() {
static const char *disposition_status_color(Disposition disposition) {
switch (disposition) {
case Disposition::kPassed: return SHELL_COLOR_GREEN();
case Disposition::kIncorrect: return SHELL_COLOR_RED();
case Disposition::kFailed: return SHELL_COLOR_RED();
default:
break;
@@ -195,16 +198,17 @@ std::ostream & PerformanceReport::print_result_pretty_(
out
<< "\n"
<< " Provider: " << SHELL_COLOR_BRIGHT() << library::to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
<< " Operation: " << result.operation_name << "\n\n"
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n"
<< " Verification: " << SHELL_COLOR_BRIGHT() << (options_.verification.enabled ? "ON":"OFF") << SHELL_COLOR_END() << "\n"
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n\n";
<< " Provider: " << SHELL_COLOR_BRIGHT() << library::to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
<< " OperationKind: " << SHELL_COLOR_BRIGHT() << library::to_string(result.op_kind) << SHELL_COLOR_END() << "\n"
<< " Operation: " << result.operation_name << "\n\n"
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n"
<< " Verification: " << SHELL_COLOR_BRIGHT() << (options_.verification.enabled ? "ON":"OFF") << SHELL_COLOR_END() << "\n"
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n\n";
// Display individual verification results for each verification-provider
if (options_.verification.enabled) {
static int const indent_spaces = 22;
static int const indent_spaces = 16;
for(auto & m : result.verification_map) {
out << std::right << std::setw(indent_spaces) << library::to_string(m.first, true) << ": " << to_string(m.second, true) << "\n";
@@ -212,15 +216,15 @@ std::ostream & PerformanceReport::print_result_pretty_(
}
out
<< "\n Arguments: ";
<< "\n Arguments:";
int column_idx = 0;
for (auto const &arg : result.arguments) {
if (!arg.second.empty()) {
out << " --" << arg.first << "=" << arg.second;
column_idx += int(4 + arg.first.size() + arg.second.size());
if (column_idx > 90) {
out << " \\\n ";
if (column_idx > 98) {
out << " \\\n ";
column_idx = 0;
}
}
@@ -228,15 +232,15 @@ std::ostream & PerformanceReport::print_result_pretty_(
out << "\n\n";
out
<< " Bytes: " << result.bytes << " bytes\n"
<< " FLOPs: " << result.flops << " flops\n\n";
<< " Bytes: " << result.bytes << " bytes\n"
<< " FLOPs: " << result.flops << " flops\n\n";
if (result.good()) {
out
<< " Runtime: " << result.runtime << " ms\n"
<< " Memory: " << result.gbytes_per_sec() << " GiB/s\n"
<< "\n Math: " << result.gflops_per_sec() << " GFLOP/s\n";
<< " Runtime: " << result.runtime << " ms\n"
<< " Memory: " << result.gbytes_per_sec() << " GiB/s\n"
<< "\n Math: " << result.gflops_per_sec() << " GFLOP/s\n";
}
@@ -256,7 +260,7 @@ std::ostream & PerformanceReport::print_csv_header_(
out
<< (column_idx ? "," : "") << "Problem,Provider"
<< ",Operation,Disposition,Status";
<< ",OperationKind,Operation,Disposition,Status";
for (auto const &arg_name : argument_names_) {
out << "," << arg_name;
@@ -289,6 +293,7 @@ std::ostream & PerformanceReport::print_result_csv_(
<< (column_idx ? "," : "")
<< result.problem_index
<< "," << to_string(result.provider, true)
<< "," << to_string(result.op_kind)
<< "," << result.operation_name
<< "," << to_string(result.disposition)
<< "," << library::to_string(result.status);
tools/profiler/src/performance_report.h
+1 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
tools/profiler/src/performance_result.cu
+55
View File
@@ -0,0 +1,55 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief
*/
#pragma once
#include <vector>
#include "cutlass/cutlass.h"
// CUTLASS Profiler includes
#include "enumerated_types.h"
#include "performance_result.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/profiler/src/performance_result.h
+3 -2
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -65,7 +65,7 @@ struct PerformanceResult {
/// Outcome of verification (all verification results)
DispositionMap verification_map;
/// Operation object
/// Operation name
std::string operation_name;
/// Stringified vector of argument values
@@ -119,3 +119,4 @@ using PerformanceResultVector = std::vector<PerformanceResult>;
} // namespace profiler
} // namespace cutlass
tools/profiler/src/problem_space.cpp
+42 -2
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -849,6 +849,47 @@ bool arg_as_OpcodeClassID(
return arg_as_OpcodeClassID(opcode_class, value_ptr);
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_SplitKModeID(
library::SplitKMode &split_k_mode,
KernelArgument::Value const *value_ptr) {
if (value_ptr->not_null) {
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
split_k_mode = library::from_string<library::SplitKMode>(
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
if (split_k_mode == library::SplitKMode::kInvalid) {
throw std::runtime_error(
"arg_as_SplitKModeID() - illegal cast.");
}
}
else {
throw std::runtime_error(
"arg_as_SplitKModeID() - illegal cast.");
}
return true;
}
return false;
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_SplitKModeID(
library::SplitKMode &split_k_mode,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
size_t idx = problem_space.argument_index(name);
KernelArgument::Value const *value_ptr = problem.at(idx).get();
return arg_as_SplitKModeID(split_k_mode, value_ptr);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
bool arg_as_scalar(
@@ -939,7 +980,6 @@ bool tensor_description_satisfies(
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
tools/profiler/src/problem_space.h
+12 -1
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
@@ -811,6 +811,17 @@ bool arg_as_OpcodeClassID(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_SplitKModeID(library::SplitKMode &split_k_mode, KernelArgument::Value const *value_ptr);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_SplitKModeID(
library::SplitKMode &split_k_mode,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
bool arg_as_scalar(
std::vector<uint8_t> &bytes,