leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

CUTLASS 2.1 (#83)

Browse Source 
CUTLASS 2.1 contributes:
- BLAS-style host-side API added to CUTLASS Library
- Planar Complex GEMM kernels targeting Volta and Turing Tensor Cores
- Minor enhancements and bug fixes
This commit is contained in:
Andrew Kerr
2020-04-07 13:51:25 -07:00
committed by GitHub
parent 7c0cd26d13
commit 96dab34ad9
196 changed files with 20653 additions and 1995 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tools/profiler/CMakeLists.txt
View File

@@ -33,7 +33,7 @@ set(CUTLASS_TOOLS_PROFILER_SOURCES
src/gpu_timer.cpp
src/device_allocation.cu
src/device_context.cu
src/cublas_helpers.cpp
src/cublas_helpers.cpp
src/problem_space.cpp
src/operation_profiler.cu
src/gemm_operation_profiler.cu
@@ -54,11 +54,11 @@ set_target_properties(cutlass_profiler PROPERTIES EXPORT_NAME profiler)
# Include paths
#
target_include_directories(cutlass_profiler
target_include_directories(
cutlass_profiler
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src # Source directory
../../tools/util/include
)
)
#
# Library dependencies
@@ -68,8 +68,8 @@ target_link_libraries(
cutlass_profiler
PRIVATE
cutlass_lib
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:cublas>
gtest
cutlass_tools_util_includes
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:nvidia::cublas>
cudart
)

23
tools/profiler/src/cublas_helpers.cpp
View File

@@ -39,14 +39,14 @@ namespace profiler {
/// Converts a cuBLAS status to cutlass::Status
Status get_cutlass_status(cublasStatus_t cublas) {
if (cublas == CUBLAS_STATUS_SUCCESS) {
return Status::kSuccess;
}
else if (cublas == CUBLAS_STATUS_INVALID_VALUE) {
return Status::kErrorInvalidProblem;
}
if (cublas == CUBLAS_STATUS_NOT_SUPPORTED) {
return Status::kErrorNotSupported;
switch (cublas) {
case CUBLAS_STATUS_SUCCESS:
return Status::kSuccess;
case CUBLAS_STATUS_INVALID_VALUE:
return Status::kErrorInvalidProblem;
case CUBLAS_STATUS_NOT_SUPPORTED:
return Status::kErrorNotSupported;
default: break;
}
return Status::kErrorInternal;
}
@@ -145,6 +145,13 @@ Status cublas_satisfies(library::GemmDescription const &desc) {
return Status::kErrorNotSupported;
}
// output type S4 and S8 not supported in cuBLAS
if (desc.C.element == library::NumericTypeID::kS4 ||
desc.C.element == library::NumericTypeID::kS8) {
return Status::kErrorNotSupported;
}
return Status::kSuccess;
}

157
tools/profiler/src/cublas_helpers.h
View File

@@ -33,7 +33,7 @@
#include "cutlass/cutlass.h"
#include "cutlass/library/library.h"
#include "options.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -86,6 +86,161 @@ public:
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Selects one or more cuBLAS algorithms.
static void select_cublas_algorithms(
std::vector<cublasGemmAlgo_t> &algorithms,
Options const &options,
library::GemmDescription const &op_desc) {
library::OpcodeClassID const & opcode_class =
op_desc.tile_description.math_instruction.opcode_class;
switch (options.library.algorithm_mode) {
case AlgorithmMode::kMatching:
{
algorithms.push_back(get_cublas_gemm_algo(
op_desc.tile_description.threadblock_shape.m(),
op_desc.tile_description.threadblock_shape.n(),
op_desc.tile_description.threadblock_shape.k(),
opcode_class));
break;
}
case AlgorithmMode::kBest:
{
// Choose first enumerated mode. If none are enumerated, choose based on opcode class
// and evaluate all of them.
if (options.library.algorithms.empty()) {
// Enumerate all algorithms
if (opcode_class == library::OpcodeClassID::kSimt) {
for (int algo = CUBLAS_GEMM_DEFAULT;
algo <= CUBLAS_GEMM_ALGO23;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
else {
for (int algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP;
algo <= CUBLAS_GEMM_ALGO15_TENSOR_OP;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
}
else {
// Use the listed algorithms
algorithms.reserve(options.library.algorithms.size());
for (int algo : options.library.algorithms) {
algorithms.push_back(reinterpret_cast<cublasGemmAlgo_t const &>(algo));
}
}
break;
}
case AlgorithmMode::kDefault:
{
// Use the library's default algorithm
algorithms.push_back((opcode_class == library::OpcodeClassID::kSimt ?
CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP));
break;
}
default:
{
break;
}
}
}
/// Dispatcher to cublasGemmEx()
struct cublasGemmExDispatcher {
//
// Data members
//
library::GemmConfiguration configuration;
library::GemmArguments arguments;
// cublass-specific data structures to fill cublas API call arguments
cublasOperation_t trans_A;
cublasOperation_t trans_B;
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_C;
cudaDataType_t compute_type;
cublasGemmAlgo_t algo;
Status status;
//
// Methods
//
cublasGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmConfiguration configuration_,
library::GemmArguments arguments_,
cublasGemmAlgo_t algorithm = CUBLAS_GEMM_DFALT
):
configuration(configuration_), arguments(arguments_), algo(algorithm), status(Status::kSuccess) {
trans_A = get_cublas_transpose_operation(op_desc.A.layout);
trans_B = get_cublas_transpose_operation(op_desc.B.layout);
bool good = true;
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
good = (good && get_cublas_datatype(
compute_type,
op_desc.tile_description.math_instruction.element_accumulator));
if (!good) {
status = Status::kErrorNotSupported;
}
}
/// Executes GEMM using these arguments
cublasStatus_t operator()(cublasHandle_t handle) {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
compute_type,
algo
);
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
} // namespace profiler
} // namespace cutlass

13
tools/profiler/src/cutlass_profiler.cu
View File

@@ -29,14 +29,9 @@
#include <iostream>
#include <stdexcept>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "cutlass_profiler.h"
#include "gemm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -49,7 +44,8 @@ CutlassProfiler::CutlassProfiler(
):
options_(options) {
operation_profilers_.emplace_back(new GemmOperationProfiler);
operation_profilers_.emplace_back(new GemmOperationProfiler);
}
CutlassProfiler::~CutlassProfiler() {
@@ -112,7 +108,7 @@ void CutlassProfiler::enumerate_() {
/// Profiles all operations
int CutlassProfiler::profile_() {
library::Manifest manifest;
library::Manifest manifest(library::Provider::kCUTLASS);
Status status = manifest.initialize();
if (status != Status::kSuccess) {
@@ -165,7 +161,8 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
}
out << "\n\nFor details about a particular function, specify the function name with --help.\n\nExample:\n\n"
<< " $ cutlass_profiler --operation=Gemm --help\n\n";
<< " $ cutlass_profiler --operation=Gemm --help\n\n"
;
}
/// Prints usage

3
tools/profiler/src/cutlass_profiler.h
View File

@@ -27,6 +27,9 @@
*/
#pragma once
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/manifest.h"
#include "options.h"
#include "operation_profiler.h"

6
tools/profiler/src/debug.h
View File

@@ -30,11 +30,11 @@
#include <iostream>
#define report(x) { std::cout << "\033[31m" << __FILE__ << ":" << __LINE__ << " " << x << "\033[0m" << std::endl; }
//#define report(x) { std::cout << "\033[31m" << __FILE__ << ":" << __LINE__ << " " << x << "\033[0m" << std::endl; }
//#define report(x) {}
// Enable/Disble Profiler debug prints
#define DEBUG_PROFILER
//#define DEBUG_PROFILER
//RED 31m // profiler prints debug messages in red
//YELLOW 33m // ir prints debug messages in yellow
@@ -43,7 +43,7 @@
#define debugprof(...)
#else
#define debugprof(...) do { \
printf("\033[31m[DEBUG PROF] %s:%d | ", __FILE__, __LINE__); \
printf("\033[33m[DEBUG PROF] %s:%d | ", __FILE__, __LINE__); \
printf(__VA_ARGS__); \
printf("\033[0m\n"); \
} while (0)

66
tools/profiler/src/device_allocation.cu
View File

@@ -34,12 +34,12 @@
#include "cutlass/util/reference/device/tensor_compare.h"
#include "cutlass/util/reference/device/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/library/util.h"
#include "device_allocation.h"
namespace cutlass {
@@ -106,6 +106,18 @@ std::vector<int> DeviceAllocation::get_packed_layout(
case library::LayoutTypeID::kRowMajorInterleavedK16:
stride = get_packed_layout_stride<cutlass::layout::RowMajorInterleaved<16>>(extent);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK32:
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<32>>(extent);
break;
case library::LayoutTypeID::kRowMajorInterleavedK32:
stride = get_packed_layout_stride<cutlass::layout::RowMajorInterleaved<32>>(extent);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK64:
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<64>>(extent);
break;
case library::LayoutTypeID::kRowMajorInterleavedK64:
stride = get_packed_layout_stride<cutlass::layout::RowMajorInterleaved<64>>(extent);
break;
case library::LayoutTypeID::kTensorNCHW:
stride = get_packed_layout_stride<cutlass::layout::TensorNCHW>(extent);
break;
@@ -200,6 +212,18 @@ size_t DeviceAllocation::construct_layout(
case library::LayoutTypeID::kRowMajorInterleavedK16:
return construct_layout_<cutlass::layout::RowMajorInterleaved<16>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kColumnMajorInterleavedK32:
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<32>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kRowMajorInterleavedK32:
return construct_layout_<cutlass::layout::RowMajorInterleaved<32>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kColumnMajorInterleavedK64:
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<64>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kRowMajorInterleavedK64:
return construct_layout_<cutlass::layout::RowMajorInterleaved<64>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kTensorNCHW:
return construct_layout_<cutlass::layout::TensorNHWC>(bytes, layout_id, extent, stride);
@@ -415,6 +439,14 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCF64:
cutlass::reference::device::BlockFillRandom<complex<double>>(
reinterpret_cast<complex<double> *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::device::BlockFillRandom<int8_t>(
reinterpret_cast<int8_t *>(pointer_),
@@ -508,6 +540,14 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCF16:
cutlass::reference::host::BlockFillRandom<cutlass::complex<cutlass::half_t>>(
reinterpret_cast<cutlass::complex<cutlass::half_t> *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF64:
cutlass::reference::host::BlockFillRandom<double>(
reinterpret_cast<double *>(host_data.data()),
@@ -516,6 +556,14 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCF64:
cutlass::reference::host::BlockFillRandom<cutlass::complex<double>>(
reinterpret_cast<cutlass::complex<double> *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::host::BlockFillRandom<int8_t>(
reinterpret_cast<int8_t *>(host_data.data()),
@@ -607,13 +655,25 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<float const *>(ptr_A),
reinterpret_cast<float const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF16:
return reference::device::BlockCompareEqual<complex<half_t>>(
reinterpret_cast<complex<half_t> const *>(ptr_A),
reinterpret_cast<complex<half_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kF64:
return reference::device::BlockCompareEqual<double>(
reinterpret_cast<double const *>(ptr_A),
reinterpret_cast<double const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF64:
return reference::device::BlockCompareEqual<complex<double>>(
reinterpret_cast<complex<double> const *>(ptr_A),
reinterpret_cast<complex<double> const *>(ptr_B),
capacity);
case library::NumericTypeID::kS8:
return reference::device::BlockCompareEqual<int8_t>(
reinterpret_cast<int8_t const *>(ptr_A),

26
tools/profiler/src/device_context.cu
View File

@@ -74,16 +74,34 @@ DeviceAllocation *DeviceContext::allocate_tensor(
allocate_tensor(name, type, layout_id, extent, stride);
if (options.initialization.enabled) {
Distribution data_distribution = options.initialization.data_distribution;
if (options.initialization.provider == Provider::kReferenceDevice) {
// check if data distribution is allowed to change
if(!options.initialization.fix_data_distribution) {
// change data distribution based on bit width
switch(type) {
case library::NumericTypeID::kB1:
data_distribution.set_uniform(0, 2, 0);
break;
case library::NumericTypeID::kS8:
data_distribution.set_uniform(-2, 2, 0);
break;
case library::NumericTypeID::kU8:
data_distribution.set_uniform(0, 4, 0);
break;
default: break;
}
}
if (options.initialization.provider == library::Provider::kReferenceDevice) {
allocation->initialize_random_device(
options.initialization.seed,
options.initialization.data_distribution);
data_distribution);
}
else if (options.initialization.provider == Provider::kReferenceHost) {
else if (options.initialization.provider == library::Provider::kReferenceHost) {
allocation->initialize_random_host(
options.initialization.seed,
options.initialization.data_distribution);
data_distribution);
}
}

48
tools/profiler/src/enumerated_types.cpp
View File

@@ -123,53 +123,6 @@ AlgorithmMode from_string<AlgorithmMode>(std::string const &str) {
/////////////////////////////////////////////////////////////////////////////////////////////////
static struct {
char const *text;
char const *pretty;
Provider enumerant;
}
Provider_enumerants[] = {
{"cutlass", "CUTLASS", Provider::kCUTLASS},
{"host", "reference_host", Provider::kReferenceHost},
{"device", "reference_device", Provider::kReferenceDevice},
{"cublas", "cuBLAS", Provider::kCUBLAS},
};
/// Converts a Provider enumerant to a string
char const *to_string(Provider provider, bool pretty) {
for (auto const & possible : Provider_enumerants) {
if (provider == possible.enumerant) {
if (pretty) {
return possible.pretty;
}
else {
return possible.text;
}
}
}
return pretty ? "Invalid" : "invalid";
}
/// Parses a Provider enumerant from a string
template <>
Provider from_string<Provider>(std::string const &str) {
for (auto const & possible : Provider_enumerants) {
if ((str.compare(possible.text) == 0) ||
(str.compare(possible.pretty) == 0)) {
return possible.enumerant;
}
}
return Provider::kInvalid;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
static struct {
char const *text;
char const *pretty;
@@ -180,6 +133,7 @@ Disposition_enumerants[] = {
{"failed", "Failed", Disposition::kFailed},
{"not_run", "Not run", Disposition::kNotRun},
{"not_verified", "Not verified", Disposition::kNotVerified},
{"invalid_problem", "Invalid problem", Disposition::kInvalidProblem},
{"not_supported", "Not supported", Disposition::kNotSupported},
{"incorrect", "Incorrect", Disposition::kIncorrect}
};

40
tools/profiler/src/enumerated_types.h
View File

@@ -30,7 +30,9 @@
#include <string>
#include <vector>
#include <map>
#include <iostream>
#include "cutlass/library/library.h"
#define TRACE(x) { std::cout << __FILE__ << ":" << __LINE__ << " " << x << std::endl; }
@@ -79,26 +81,6 @@ AlgorithmMode from_string<AlgorithmMode>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Providers
enum class Provider {
kCUTLASS,
kReferenceHost,
kReferenceDevice,
kCUBLAS,
kInvalid
};
using ProviderVector = std::vector<Provider>;
/// Converts a Provider enumerant to a string
char const *to_string(Provider provider, bool pretty = false);
/// Parses a Provider enumerant from a string
template <>
Provider from_string<Provider>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Outcome of a performance test
enum class Disposition {
kPassed,
@@ -106,12 +88,13 @@ enum class Disposition {
kNotRun,
kIncorrect,
kNotVerified,
kInvalidProblem,
kNotSupported,
kInvalid
};
/// Converts a Disposition enumerant to a string
char const *to_string(Disposition provider, bool pretty = false);
char const *to_string(Disposition disposition, bool pretty = false);
/// Parses a Disposition enumerant from a string
template <>
@@ -159,6 +142,21 @@ char const *to_string(ArgumentTypeID type, bool pretty = false);
template <>
ArgumentTypeID from_string<ArgumentTypeID>(std::string const &str);
/////////////////////////////////////////////////////////////////////////////////////////////////
// Profiler typedefs
using ProviderVector = std::vector<library::Provider>;
using DispositionMap = std::map<library::Provider, Disposition>;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Print vector for the report
template <typename T>
std::ostream& operator<< (std::ostream& out, const std::vector<T>& v) {
for(int i = 0; i < v.size(); ++i) {
out << to_string(v[i], true) << (i+1 != v.size() ? "," : "");
}
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler

267
tools/profiler/src/gemm_operation_profiler.cu
View File

@@ -140,7 +140,6 @@ Status GemmOperationProfiler::initialize_configuration(
return Status::kErrorInvalidProblem;
}
if (!arg_as_int(problem_.m, "m", problem_space, problem)) {
// default value
problem_.m = 1024;
@@ -201,7 +200,7 @@ Status GemmOperationProfiler::initialize_configuration(
return Status::kErrorInternal;
}
}
problem_.lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout, {int(problem_.m), int(problem_.k)}).front();
@@ -240,7 +239,7 @@ void GemmOperationProfiler::initialize_result_(
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space) {
result.provider = Provider::kCUTLASS;
result.provider = library::Provider::kCUTLASS;
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
@@ -277,9 +276,17 @@ void GemmOperationProfiler::initialize_result_(
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n * 2;
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
result.runtime = 0;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
case library::MathOperationID::kMultiplyAddComplex:
result.flops *= 4;
break;
default: break;
}
}
/// Initializes workspace
@@ -290,7 +297,7 @@ Status GemmOperationProfiler::initialize_workspace(
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
@@ -348,7 +355,7 @@ Status GemmOperationProfiler::initialize_workspace(
//
Status status = Status::kSuccess;
if (options.profiling.provider_enabled(Provider::kCUTLASS)) {
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
if (options.execution_mode != ExecutionMode::kDryRun) {
@@ -368,8 +375,12 @@ Status GemmOperationProfiler::initialize_workspace(
// If CUTLASS is enabled, generate a result for it
//
results_.push_back(model_result_);
results_.back().provider = Provider::kCUTLASS;
results_.back().provider = library::Provider::kCUTLASS;
results_.back().op_kind = library::OperationKind::kGemm;
results_.back().disposition = Disposition::kNotRun;
for(auto &verification_provider : options.verification.providers) {
results_.back().verification_map[verification_provider] = Disposition::kNotRun;
}
}
return status;
@@ -386,7 +397,7 @@ bool GemmOperationProfiler::verify_cutlass(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!options.profiling.provider_enabled(Provider::kCUTLASS)) {
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
return true;
}
@@ -423,198 +434,62 @@ bool GemmOperationProfiler::verify_cutlass(
return false;
}
// CUTLASS op ran the but not yet verified against any verification provider
results_.back().disposition = Disposition::kNotVerified;
//
// Run verification providers
//
if (options.verification.enabled) {
#if CUTLASS_ENABLE_CUBLAS
if (options.verification.provider_enabled(Provider::kCUBLAS)) {
if (options.verification.provider_enabled(library::Provider::kCUBLAS)) {
// Guard against unsupported cases
auto const & gemm_desc = static_cast<library::GemmDescription const &>(operation->description());
if (cublas_satisfies(gemm_desc) != Status::kSuccess) {
return true;
}
if (cublas_satisfies(gemm_desc) == Status::kSuccess) {
return verify_with_cublas_(
options,
report,
device_context,
operation,
problem_space,
problem);
// call cublas verification if supported
verify_with_cublas_(
options,
report,
device_context,
operation,
problem_space,
problem);
}
else {
// set verification map for cublas to not supported
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kNotSupported;
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
results_.back().disposition = m.second;
return true;
}
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
is_any_verification_run_passed = true;
}
}
if(is_any_verification_run_passed) {
results_.back().disposition = Disposition::kPassed;
}
}
// Return true means continue profiling
return true;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#if CUTLASS_ENABLE_CUBLAS
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Selects one or more cuBLAS algorithms.
static void select_cublas_algorithms(
std::vector<cublasGemmAlgo_t> &algorithms,
Options const &options,
library::GemmDescription const &op_desc) {
library::OpcodeClassID const & opcode_class =
op_desc.tile_description.math_instruction.opcode_class;
switch (options.library.algorithm_mode) {
case AlgorithmMode::kMatching:
{
algorithms.push_back(get_cublas_gemm_algo(
op_desc.tile_description.threadblock_shape.m(),
op_desc.tile_description.threadblock_shape.n(),
op_desc.tile_description.threadblock_shape.k(),
opcode_class));
break;
}
case AlgorithmMode::kBest:
{
// Choose first enumerated mode. If none are enumerated, choose based on opcode class
// and evaluate all of them.
if (options.library.algorithms.empty()) {
// Enumerate all algorithms
if (opcode_class == library::OpcodeClassID::kSimt) {
for (int algo = CUBLAS_GEMM_DEFAULT;
algo <= CUBLAS_GEMM_ALGO23;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
else {
for (int algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP;
algo <= CUBLAS_GEMM_ALGO15_TENSOR_OP;
++algo) {
algorithms.push_back(cublasGemmAlgo_t(algo));
}
}
}
else {
// Use the listed algorithms
algorithms.reserve(options.library.algorithms.size());
for (int algo : options.library.algorithms) {
algorithms.push_back(reinterpret_cast<cublasGemmAlgo_t const &>(algo));
}
}
break;
}
case AlgorithmMode::kDefault:
{
// Use the library's default algorithm
algorithms.push_back((opcode_class == library::OpcodeClassID::kSimt ?
CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP));
break;
}
default:
{
break;
}
}
}
/// Dispatcher to cublasGemmEx()
struct cublasGemmExDispatcher {
//
// Data members
//
library::GemmConfiguration configuration;
library::GemmArguments arguments;
cublasOperation_t trans_A;
cublasOperation_t trans_B;
cudaDataType_t data_type_A;
cudaDataType_t data_type_B;
cudaDataType_t data_type_C;
cudaDataType_t compute_type;
cublasGemmAlgo_t algo;
Status status;
//
// Methods
//
cublasGemmExDispatcher(
library::GemmDescription const &op_desc,
library::GemmConfiguration configuration_,
library::GemmArguments arguments_,
cublasGemmAlgo_t algorithm = CUBLAS_GEMM_DFALT
):
configuration(configuration_), arguments(arguments_), algo(algorithm), status(Status::kSuccess) {
trans_A = get_cublas_transpose_operation(op_desc.A.layout);
trans_B = get_cublas_transpose_operation(op_desc.B.layout);
bool good = true;
good = (good && get_cublas_datatype(data_type_A, op_desc.A.element));
good = (good && get_cublas_datatype(data_type_B, op_desc.B.element));
good = (good && get_cublas_datatype(data_type_C, op_desc.C.element));
good = (good && get_cublas_datatype(
compute_type,
op_desc.tile_description.math_instruction.element_accumulator));
if (!good) {
status = Status::kErrorNotSupported;
}
}
/// Executes GEMM using these arguments
cublasStatus_t operator()(cublasHandle_t handle) {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
compute_type,
algo
);
}
};
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
#endif // CUTLASS_ENABLE_CUBLAS
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Verifies CUTLASS against references
@@ -632,14 +507,16 @@ bool GemmOperationProfiler::verify_with_cublas_(
library::GemmDescription const &gemm_desc =
static_cast<library::GemmDescription const &>(operation->description());
//
// Construct cuBLAS operators
//
CublasCreate handle;
cublasStatus_t status = handle.get_cublas_create_status();
if (status != CUBLAS_STATUS_SUCCESS) {
results_.back().status = get_cutlass_status(status);
results_.back().disposition = Disposition::kFailed;
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
return true;
}
@@ -682,7 +559,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
);
if (gemm_op.status != Status::kSuccess) {
results_.back().disposition = Disposition::kNotVerified;
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
return true;
}
@@ -692,8 +570,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Handle errors
if (status != CUBLAS_STATUS_SUCCESS) {
results_.back().status = get_cutlass_status(status);
results_.back().disposition = Disposition::kNotVerified;
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
return true;
}
@@ -701,7 +579,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Verify results
//
results_.back().disposition = compare_tensors(
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
options,
*gemm_workspace_.Computed,
*gemm_workspace_.Reference
@@ -709,19 +587,18 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().disposition == Disposition::kIncorrect) {
results_.back().verification_map[library::Provider::kCUBLAS] == Disposition::kIncorrect) {
save_workspace(
device_context,
options,
gemm_desc,
Provider::kCUTLASS,
Provider::kCUBLAS);
library::Provider::kCUTLASS,
library::Provider::kCUBLAS);
}
}
catch (...) {
results_.back().disposition = Disposition::kFailed;
results_.back().status = Status::kErrorNotSupported;
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
}
#endif
@@ -741,7 +618,7 @@ bool GemmOperationProfiler::profile(
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.profiling.provider_enabled(Provider::kCUTLASS)) {
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();

1
tools/profiler/src/gemm_operation_profiler.h
View File

@@ -35,6 +35,7 @@
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes

19
tools/profiler/src/operation_profiler.cu
View File

@@ -225,7 +225,7 @@ int OperationProfiler::profile_all(
ProblemSpace problem_space(arguments_, options.cmdline);
// 1. Construct performance report
PerformanceReport report(options, problem_space.argument_names());
PerformanceReport report(options, problem_space.argument_names(), kind_);
// 2. For each problem in problem space
ProblemSpace::Iterator problem_it = problem_space.begin();
@@ -269,7 +269,7 @@ int OperationProfiler::profile_all(
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
continue;
}
// A. Initialize configuration
Status status = this->initialize_configuration(
options,
@@ -278,7 +278,7 @@ int OperationProfiler::profile_all(
operation,
problem_space,
problem);
if (status == Status::kErrorInternal) {
// Stop profiling if there was an internal error
return false;
@@ -341,7 +341,7 @@ int OperationProfiler::profile_all(
device_context,
options,
operation->description(),
Provider::kCUTLASS);
library::Provider::kCUTLASS);
}
//
@@ -434,8 +434,8 @@ void OperationProfiler::save_workspace(
DeviceContext &device_context,
Options const &options,
library::OperationDescription const &desc,
Provider provider,
Provider verification_provider) {
library::Provider provider,
library::Provider verification_provider) {
for (auto const & named_allocation : device_context) {
@@ -443,10 +443,10 @@ void OperationProfiler::save_workspace(
std::stringstream filename;
filename << desc.name << "_" << to_string(provider) << "_";
filename << desc.name << "_" << library::to_string(provider) << "_";
if (verification_provider != Provider::kInvalid) {
filename << "verified_by_" << to_string(verification_provider) << "_";
if (verification_provider != library::Provider::kInvalid) {
filename << "verified_by_" << library::to_string(verification_provider) << "_";
}
filename << named_allocation.first + ".mat";
@@ -454,6 +454,7 @@ void OperationProfiler::save_workspace(
std::ofstream out(filename.str());
allocation->write_tensor_csv(out);
out << "\n";
if (options.report.verbose) {
std::cout << "wrote '" << filename.str() << "'" << std::endl;

6
tools/profiler/src/operation_profiler.h
View File

@@ -35,6 +35,7 @@
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
@@ -43,6 +44,7 @@
#include "performance_result.h"
#include "performance_report.h"
#include "problem_space.h"
#include "debug.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -192,8 +194,8 @@ public:
DeviceContext &device_context,
Options const &options,
library::OperationDescription const &desc,
Provider provider,
Provider verification_provider = Provider::kInvalid);
library::Provider provider,
library::Provider verification_provider = library::Provider::kInvalid);
protected:

42
tools/profiler/src/options.cu
View File

@@ -31,6 +31,8 @@
#include "cutlass/cutlass.h"
#include "cutlass/version.h"
#include "cutlass/library/util.h"
#include "options.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -161,24 +163,30 @@ Options::Initialization::Initialization(cutlass::CommandLine const &cmdline) {
if (cmdline.check_cmd_line_flag("initialization-provider")) {
std::string str;
cmdline.get_cmd_line_argument("initialization-provider", str);
provider = from_string<Provider>(str);
if (provider == Provider::kInvalid) {
provider = library::from_string<library::Provider>(str);
if (provider == library::Provider::kInvalid) {
enabled = false;
}
else if (provider != Provider::kReferenceHost && provider != Provider::kReferenceDevice) {
else if (provider != library::Provider::kReferenceHost && provider != library::Provider::kReferenceDevice) {
throw std::runtime_error("Unsupported intialization provider specified.");
}
}
else {
provider = Provider::kReferenceDevice;
provider = library::Provider::kReferenceDevice;
}
cmdline.get_cmd_line_argument("seed", seed, 2019);
if (cmdline.check_cmd_line_flag("dist")) {
// user has set the data distribution (fix data distribution once set)
fix_data_distribution = true;
// set user provided data distribution
get_distribution(cmdline, "dist", data_distribution);
}
else {
// profiler choosen data distribution (allowed to change based on numeric types)
fix_data_distribution = false;
// set uniform data distribution with range [-4, 4]
data_distribution.set_uniform(-4, 4, 0);
}
@@ -372,12 +380,12 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
providers.clear();
for (auto const &token : tokens) {
providers.push_back(from_string<Provider>(token));
providers.push_back(library::from_string<library::Provider>(token));
}
}
else {
providers.push_back(Provider::kCUTLASS);
providers.push_back(Provider::kCUBLAS);
providers.push_back(library::Provider::kCUTLASS);
providers.push_back(library::Provider::kCUBLAS);
}
}
@@ -412,18 +420,18 @@ void Options::Profiling::print_options(std::ostream &out, int indent) const {
int j = 0;
for (auto const & provider : providers) {
out << (j++ ? ", " : "") << to_string(provider);
out << (j++ ? ", " : "") << library::to_string(provider);
}
out << "]\n";
}
/// Returns true if a provider is enabled
bool Options::Profiling::provider_enabled(Provider provider) const {
bool Options::Profiling::provider_enabled(library::Provider provider) const {
return std::find(providers.begin(), providers.end(), provider) != providers.end();
}
/// Returns the index of a provider if its enabled
size_t Options::Profiling::index(Provider provider) const {
size_t Options::Profiling::index(library::Provider provider) const {
size_t idx = 0;
for (auto const & x : providers) {
if (x == provider) {
@@ -461,14 +469,14 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
providers.clear();
for (auto const &token : tokens) {
Provider provider = from_string<Provider>(token);
if (provider != Provider::kInvalid) {
library::Provider provider = library::from_string<library::Provider>(token);
if (provider != library::Provider::kInvalid) {
providers.push_back(provider);
}
}
}
else {
providers.push_back(Provider::kCUBLAS);
providers.push_back(library::Provider::kCUBLAS);
}
}
@@ -504,18 +512,18 @@ void Options::Verification::print_options(std::ostream &out, int indent) const {
int j = 0;
for (auto const & provider : providers) {
out << (j++ ? ", " : "") << to_string(provider);
out << (j++ ? ", " : "") << library::to_string(provider);
}
out << "]\n";
}
/// Returns true if a provider is enabled
bool Options::Verification::provider_enabled(Provider provider) const {
bool Options::Verification::provider_enabled(library::Provider provider) const {
return std::find(providers.begin(), providers.end(), provider) != providers.end();
}
/// Returns the index of a provider if its enabled
size_t Options::Verification::index(Provider provider) const {
size_t Options::Verification::index(library::Provider provider) const {
size_t idx = 0;
for (auto const & x : providers) {
if (x == provider) {
@@ -658,7 +666,7 @@ Options::Options(cutlass::CommandLine const &cmdline):
// Prevent launches on the device for anything other than CUTLASS operation
if (execution_mode == ExecutionMode::kTrace) {
initialization.provider = Provider::kReferenceHost;
initialization.provider = library::Provider::kReferenceHost;
verification.enabled = false;
profiling.enabled = false;
}

14
tools/profiler/src/options.h
View File

@@ -105,11 +105,15 @@ public:
/// allocating tensors.
bool enabled;
/// If true, data distribution is set by the user and is not allowed to change
/// If false, data distribution is allowed to change based on element_type (library::NumericTypeID)
bool fix_data_distribution;
/// Data distribution for input tensors
Distribution data_distribution;
/// Source of random tensor elements
Provider provider;
library::Provider provider;
/// Random number generator seed.
int seed;
@@ -162,10 +166,10 @@ public:
void print_options(std::ostream &out, int indent = 0) const;
/// Returns true if a provider is enabled
bool provider_enabled(Provider provider) const;
bool provider_enabled(library::Provider provider) const;
/// Returns the index of a provider if its enabled
size_t index(Provider provider) const;
size_t index(library::Provider provider) const;
};
/// Options related to profiling
@@ -196,10 +200,10 @@ public:
void print_options(std::ostream &out, int indent = 0) const;
/// Returns true if a provider is enabled
bool provider_enabled(Provider provider) const;
bool provider_enabled(library::Provider provider) const;
/// Returns the index of a provider if its enabled
size_t index(Provider provider) const;
size_t index(library::Provider provider) const;
};
/// Options related to reporting

60
tools/profiler/src/performance_report.cpp
View File

@@ -29,9 +29,15 @@
#include <iostream>
#include <stdexcept>
#include <iomanip>
#include <algorithm>
#include <cstring>
#include "cutlass/library/util.h"
#include "cutlass/library/util.h"
#include "performance_report.h"
#include "debug.h"
namespace cutlass {
namespace profiler {
@@ -57,12 +63,17 @@ namespace profiler {
PerformanceReport::PerformanceReport(
Options const &options,
std::vector<std::string> const &argument_names
std::vector<std::string> const &argument_names,
library::OperationKind const &op_kind
):
options_(options), argument_names_(argument_names), problem_index_(0), good_(true) {
options_(options), argument_names_(argument_names), problem_index_(0), good_(true), op_kind_(op_kind) {
std::string file_name = options_.report.output_path.substr(0, options_.report.output_path.rfind("."));
std::string file_extension = options_.report.output_path.substr(options_.report.output_path.rfind(".") + 1);
op_file_name_ = file_name + "." + to_string(op_kind_) + "." + file_extension;
//
// Open output file
// Open output file for operation of PerformanceReport::op_kind
//
if (!options_.report.output_path.empty()) {
@@ -70,17 +81,17 @@ PerformanceReport::PerformanceReport(
if (options_.report.append) {
std::ifstream test_output_file(options_.report.output_path.c_str());
std::ifstream test_output_file(op_file_name_);
if (test_output_file.is_open()) {
print_header = false;
test_output_file.close();
}
output_file_.open(options_.report.output_path.c_str(), std::ios::app);
output_file_.open(op_file_name_, std::ios::app);
}
else {
output_file_.open(options_.report.output_path.c_str());
output_file_.open(op_file_name_);
}
if (!output_file_.good()) {
@@ -148,7 +159,7 @@ void PerformanceReport::close() {
}
}
else if (output_file_.is_open() && options_.report.verbose) {
std::cout << "\n\nWrote results to '" << options_.report.output_path << "'" << std::endl;
std::cout << "\n\nWrote results to '" << op_file_name_ << "'" << std::endl;
}
}
@@ -184,19 +195,30 @@ std::ostream & PerformanceReport::print_result_pretty_(
out
<< "\n"
<< " Provider: " << SHELL_COLOR_BRIGHT() << to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
<< " Operation: " << result.operation_name << "\n\n"
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n"
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n";
<< " Provider: " << SHELL_COLOR_BRIGHT() << library::to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
<< " Operation: " << result.operation_name << "\n\n"
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n"
<< " Verification: " << SHELL_COLOR_BRIGHT() << (options_.verification.enabled ? "ON":"OFF") << SHELL_COLOR_END() << "\n"
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n\n";
// Display individual verification results for each verification-provider
if (options_.verification.enabled) {
static int const indent_spaces = 22;
for(auto & m : result.verification_map) {
out << std::right << std::setw(indent_spaces) << library::to_string(m.first, true) << ": " << to_string(m.second, true) << "\n";
}
}
out
<< "\n Arguments: ";
<< "\n Arguments: ";
int column_idx = 0;
for (auto const &arg : result.arguments) {
if (!arg.second.empty()) {
out << " --" << arg.first << "=" << arg.second;
column_idx += 4 + arg.first.size() + arg.second.size();
column_idx += int(4 + arg.first.size() + arg.second.size());
if (column_idx > 90) {
out << " \\\n ";
column_idx = 0;
@@ -206,15 +228,15 @@ std::ostream & PerformanceReport::print_result_pretty_(
out << "\n\n";
out
<< " Bytes: " << result.bytes << " bytes\n"
<< " FLOPs: " << result.flops << " flops\n\n";
<< " Bytes: " << result.bytes << " bytes\n"
<< " FLOPs: " << result.flops << " flops\n\n";
if (result.good()) {
out
<< " Runtime: " << result.runtime << " ms\n"
<< " Memory: " << result.gbytes_per_sec() << " GiB/s\n"
<< "\n Math: " << result.gflops_per_sec() << " GFLOP/s\n";
<< " Runtime: " << result.runtime << " ms\n"
<< " Memory: " << result.gbytes_per_sec() << " GiB/s\n"
<< "\n Math: " << result.gflops_per_sec() << " GFLOP/s\n";
}

12
tools/profiler/src/performance_report.h
View File

@@ -31,10 +31,14 @@
#include <vector>
#include <fstream>
// CUTLASS Profiler includes
#include "options.h"
#include "enumerated_types.h"
#include "performance_result.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
namespace cutlass {
namespace profiler {
@@ -46,6 +50,12 @@ private:
/// Reference to options
Options const &options_;
/// Operation kind
library::OperationKind op_kind_;
/// Operation file name containing performance report of op_kind
std::string op_file_name_;
/// Output file containing results
std::ofstream output_file_;
@@ -63,7 +73,7 @@ private:
public:
PerformanceReport(Options const &options, std::vector<std::string> const &argument_names);
PerformanceReport(Options const &options, std::vector<std::string> const &argument_names, library::OperationKind const &op_kind);
bool good() const { return good_; }

24
tools/profiler/src/performance_result.h
View File

@@ -32,8 +32,12 @@
#include "cutlass/cutlass.h"
// CUTLASS Profiler includes
#include "enumerated_types.h"
// CUTLASS Library includes
#include "cutlass/library/library.h"
namespace cutlass {
namespace profiler {
@@ -45,15 +49,22 @@ struct PerformanceResult {
/// Index of problem
size_t problem_index;
/// Provider
Provider provider;
/// library::Provider
library::Provider provider;
/// Outcome of test
Disposition disposition;
/// Operation kind
library::OperationKind op_kind;
/// CUTLASS status result from kernels
/// CUTLASS status result from kernels (success or failure)
// Status does information on verification
Status status;
/// Outcome of verification (worst case verification result)
Disposition disposition;
/// Outcome of verification (all verification results)
DispositionMap verification_map;
/// Operation object
std::string operation_name;
@@ -76,7 +87,8 @@ struct PerformanceResult {
/// Ctor
PerformanceResult():
problem_index(0),
provider(Provider::kInvalid),
op_kind(library::OperationKind::kInvalid),
provider(library::Provider::kInvalid),
disposition(Disposition::kNotRun),
status(Status::kInvalid),
bytes(0),

8
tools/profiler/src/problem_space.cpp
View File

@@ -27,10 +27,11 @@
*/
#include <string>
#include <iostream>
#include <stdexcept>
#include <sstream>
#include "cutlass/library/util.h"
#include "problem_space.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -849,17 +850,16 @@ bool arg_as_OpcodeClassID(
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
bool arg_as_scalar(
std::vector<uint8_t> &bytes,
library::NumericTypeID numeric_type,
KernelArgument::Value const *value_ptr) {
if (value_ptr->not_null) {
if (value_ptr->argument->description->type == ArgumentTypeID::kInteger) {
int64_t int_value = static_cast<IntegerArgument::IntegerValue const *>(value_ptr)->value;
// TODO - convert int64_t => destination type
}
else if (value_ptr->argument->description->type == ArgumentTypeID::kScalar) {