CUTLASS 2.3 initial commit (#134)

CUTLASS 2.3 adds GEMMs targeting Sparse Tensor Cores on the NVIDIA Ampere Architecture, fast SGEMM, and small matrix classes, bug fixes, and performance enhancements.
This commit is contained in:
Andrew Kerr
2020-09-23 14:00:58 -07:00
committed by GitHub
parent 4dac7490e6
commit c53f3339bb
209 changed files with 46922 additions and 1677 deletions
+71 -28
View File
@@ -51,6 +51,18 @@ Status get_cutlass_status(cublasStatus_t cublas) {
return Status::kErrorInternal;
}
/// Converts a cuBLASS status to cutlass::profiler::Disposition
Disposition get_cutlass_disposition(cublasStatus_t cublas_status) {
if (cublas_status == CUBLAS_STATUS_INVALID_VALUE) {
return Disposition::kInvalidProblem;
}
else if (cublas_status == CUBLAS_STATUS_NOT_SUPPORTED) {
return Disposition::kNotSupported;
}
return Disposition::kFailed;
}
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
bool get_cublas_transpose_operation(
cublasOperation_t &operation,
@@ -156,7 +168,6 @@ bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID ele
/// Gets the cublas algorithm given threadblock tile dimensions and math opcode class
cublasGemmAlgo_t get_cublas_gemm_algo(int cta_m, int cta_n, int cta_k, library::OpcodeClassID opcode_class) {
// TODO
return (opcode_class == library::OpcodeClassID::kSimt ?
CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP);
}
@@ -252,37 +263,69 @@ cublasGemmExDispatcher::cublasGemmExDispatcher(
/// Executes GEMM using these arguments
cublasStatus_t cublasGemmExDispatcher::operator()(cublasHandle_t handle) {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
#if (__CUDA_VER_MAJOR__ >= 11)
compute_type,
#else
compute_data_type,
#endif
algo
);
if (configuration.mode == library::GemmUniversalMode::kBatched) {
return cublasGemmStridedBatchedEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.batch_stride_A,
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.batch_stride_B,
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
arguments.batch_stride_C,
configuration.batch_count,
#if (__CUDA_VER_MAJOR__ >= 11)
compute_type,
#else
compute_data_type,
#endif
algo
);
}
else {
return cublasGemmEx(
handle,
trans_A,
trans_B,
configuration.problem_size.m(),
configuration.problem_size.n(),
configuration.problem_size.k(),
arguments.alpha,
arguments.A,
data_type_A,
int(configuration.lda),
arguments.B,
data_type_B,
int(configuration.ldb),
arguments.beta,
arguments.D,
data_type_C,
int(configuration.ldc),
#if (__CUDA_VER_MAJOR__ >= 11)
compute_type,
#else
compute_data_type,
#endif
algo
);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
+3
View File
@@ -47,6 +47,9 @@ namespace profiler {
/// Converts a cuBLAS status to cutlass::Status
Status get_cutlass_status(cublasStatus_t cublas);
/// Converts a cuBLASS status to cutlass::profiler::Disposition
Disposition get_cutlass_disposition(cublasStatus_t cublas_status);
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
bool get_cublas_transpose_operation(
cublasOperation_t &operation,
+4
View File
@@ -32,6 +32,8 @@
// Profiler includes
#include "cutlass_profiler.h"
#include "gemm_operation_profiler.h"
#include "sparse_gemm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -46,6 +48,8 @@ CutlassProfiler::CutlassProfiler(
operation_profilers_.emplace_back(new GemmOperationProfiler(options));
operation_profilers_.emplace_back(new SparseGemmOperationProfiler(options));
}
CutlassProfiler::~CutlassProfiler() {
+455 -15
View File
@@ -94,6 +94,12 @@ std::vector<int> DeviceAllocation::get_packed_layout(
case library::LayoutTypeID::kRowMajor:
stride = get_packed_layout_stride<cutlass::layout::RowMajor>(extent);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK2:
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<2>>(extent);
break;
case library::LayoutTypeID::kRowMajorInterleavedK2:
stride = get_packed_layout_stride<cutlass::layout::RowMajorInterleaved<2>>(extent);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK4:
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<4>>(extent);
break;
@@ -124,7 +130,9 @@ std::vector<int> DeviceAllocation::get_packed_layout(
case library::LayoutTypeID::kTensorNHWC:
stride = get_packed_layout_stride<cutlass::layout::TensorNHWC>(extent);
break;
case library::LayoutTypeID::kTensorNDHWC:
stride = get_packed_layout_stride<cutlass::layout::TensorNDHWC>(extent);
break;
default: break;
}
@@ -200,6 +208,12 @@ size_t DeviceAllocation::construct_layout(
case library::LayoutTypeID::kRowMajor:
return construct_layout_<cutlass::layout::RowMajor>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kColumnMajorInterleavedK2:
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<2>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kRowMajorInterleavedK2:
return construct_layout_<cutlass::layout::RowMajorInterleaved<2>>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kColumnMajorInterleavedK4:
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<4>>(bytes, layout_id, extent, stride);
@@ -230,6 +244,9 @@ size_t DeviceAllocation::construct_layout(
case library::LayoutTypeID::kTensorNHWC:
return construct_layout_<cutlass::layout::TensorNHWC>(bytes, layout_id, extent, stride);
case library::LayoutTypeID::kTensorNDHWC:
return construct_layout_<cutlass::layout::TensorNDHWC>(bytes, layout_id, extent, stride);
default: break;
}
@@ -240,9 +257,11 @@ size_t DeviceAllocation::construct_layout(
DeviceAllocation::DeviceAllocation():
type_(library::NumericTypeID::kInvalid),
batch_stride_(0),
capacity_(0),
pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown) {
layout_(library::LayoutTypeID::kUnknown),
batch_count_(1) {
}
@@ -250,7 +269,8 @@ DeviceAllocation::DeviceAllocation(
library::NumericTypeID type,
size_t capacity
):
type_(type), capacity_(capacity), pointer_(nullptr), layout_(library::LayoutTypeID::kUnknown) {
type_(type), batch_stride_(capacity), capacity_(capacity), pointer_(nullptr),
layout_(library::LayoutTypeID::kUnknown), batch_count_(1) {
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity));
@@ -266,11 +286,12 @@ DeviceAllocation::DeviceAllocation(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride
std::vector<int> const &stride,
int batch_count
):
type_(type), capacity_(size_t(0)), pointer_(nullptr) {
type_(type), batch_stride_(size_t(0)), capacity_(size_t(0)), pointer_(nullptr), batch_count_(1) {
reset(type, layout_id, extent, stride);
reset(type, layout_id, extent, stride, batch_count);
}
DeviceAllocation::~DeviceAllocation() {
@@ -285,12 +306,14 @@ DeviceAllocation &DeviceAllocation::reset() {
}
type_ = library::NumericTypeID::kInvalid;
batch_stride_ = 0;
capacity_ = 0;
pointer_ = nullptr;
layout_ = library::LayoutTypeID::kUnknown;
stride_.clear();
extent_.clear();
tensor_ref_buffer_.clear();
batch_count_ = 1;
return *this;
}
@@ -299,16 +322,19 @@ DeviceAllocation &DeviceAllocation::reset(library::NumericTypeID type, size_t ca
reset();
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity));
type_ = type;
batch_stride_ = capacity;
capacity_ = capacity;
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type_, capacity_));
if (result != cudaSuccess) {
throw std::bad_alloc();
}
type_ = type;
capacity_ = capacity;
layout_ = library::LayoutTypeID::kUnknown;
stride_.clear();
extent_.clear();
batch_count_ = 1;
tensor_ref_buffer_.resize(sizeof(pointer_), 0);
std::memcpy(tensor_ref_buffer_.data(), &pointer_, sizeof(pointer_));
@@ -321,7 +347,8 @@ DeviceAllocation &DeviceAllocation::reset(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride) {
std::vector<int> const &stride,
int batch_count) {
reset();
@@ -332,13 +359,16 @@ DeviceAllocation &DeviceAllocation::reset(
layout_ = layout_id;
stride_ = stride;
extent_ = extent;
batch_count_ = batch_count;
capacity_ = construct_layout(
batch_stride_ = construct_layout(
tensor_ref_buffer_.data() + sizeof(pointer_),
layout_id,
extent,
stride_);
capacity_ = batch_stride_ * batch_count_;
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity_));
if (result != cudaSuccess) {
throw std::bad_alloc();
@@ -361,6 +391,10 @@ void *DeviceAllocation::data() const {
return pointer_;
}
void *DeviceAllocation::batch_data(int batch_idx) const {
return static_cast<char *>(data()) + batch_stride_bytes() * batch_idx;
}
library::LayoutTypeID DeviceAllocation::layout() const {
return layout_;
}
@@ -374,6 +408,21 @@ std::vector<int> const & DeviceAllocation::extent() const {
return extent_;
}
/// Gets the number of adjacent tensors in memory
int DeviceAllocation::batch_count() const {
return batch_count_;
}
/// Gets the stride (in units of elements) beteween items
int64_t DeviceAllocation::batch_stride() const {
return batch_stride_;
}
/// Gets the stride (in units of bytes) beteween items
int64_t DeviceAllocation::batch_stride_bytes() const {
return bytes(type_, batch_stride_);
}
size_t DeviceAllocation::capacity() const {
return capacity_;
}
@@ -423,6 +472,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kBF16:
cutlass::reference::device::BlockFillRandom<cutlass::bfloat16_t>(
reinterpret_cast<cutlass::bfloat16_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kTF32:
cutlass::reference::device::BlockFillRandom<cutlass::tfloat32_t>(
reinterpret_cast<cutlass::tfloat32_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF32:
cutlass::reference::device::BlockFillRandom<float>(
reinterpret_cast<float *>(pointer_),
@@ -431,6 +496,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCBF16:
cutlass::reference::device::BlockFillRandom<complex<bfloat16_t>>(
reinterpret_cast<complex<bfloat16_t> *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kCTF32:
cutlass::reference::device::BlockFillRandom<cutlass::complex<cutlass::tfloat32_t>>(
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::device::BlockFillRandom<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(pointer_),
@@ -455,6 +536,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kS2:
cutlass::reference::device::BlockFillRandom<int2b_t>(
reinterpret_cast<int2b_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS4:
cutlass::reference::device::BlockFillRandom<int4b_t>(
reinterpret_cast<int4b_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::device::BlockFillRandom<int8_t>(
reinterpret_cast<int8_t *>(pointer_),
@@ -487,6 +584,30 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kB1:
cutlass::reference::device::BlockFillRandom<uint1b_t>(
reinterpret_cast<uint1b_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU2:
cutlass::reference::device::BlockFillRandom<uint2b_t>(
reinterpret_cast<uint2b_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU4:
cutlass::reference::device::BlockFillRandom<uint4b_t>(
reinterpret_cast<uint4b_t *>(pointer_),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU8:
cutlass::reference::device::BlockFillRandom<uint8_t>(
reinterpret_cast<uint8_t *>(pointer_),
@@ -523,7 +644,6 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
}
}
void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
if (!good()) {
throw std::runtime_error("Attempting to initialize invalid allocation.");
@@ -540,6 +660,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kBF16:
cutlass::reference::host::BlockFillRandom<cutlass::bfloat16_t>(
reinterpret_cast<cutlass::bfloat16_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kTF32:
cutlass::reference::host::BlockFillRandom<cutlass::tfloat32_t>(
reinterpret_cast<cutlass::tfloat32_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kF32:
cutlass::reference::host::BlockFillRandom<float>(
reinterpret_cast<float *>(host_data.data()),
@@ -556,6 +692,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kCBF16:
cutlass::reference::host::BlockFillRandom<cutlass::complex<cutlass::bfloat16_t>>(
reinterpret_cast<cutlass::complex<cutlass::bfloat16_t> *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kCTF32:
cutlass::reference::host::BlockFillRandom<cutlass::complex<cutlass::tfloat32_t>>(
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kCF32:
cutlass::reference::host::BlockFillRandom<cutlass::complex<float>>(
reinterpret_cast<cutlass::complex<float> *>(host_data.data()),
@@ -580,6 +732,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kS2:
cutlass::reference::host::BlockFillRandom<int2b_t>(
reinterpret_cast<int2b_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS4:
cutlass::reference::host::BlockFillRandom<int4b_t>(
reinterpret_cast<int4b_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kS8:
cutlass::reference::host::BlockFillRandom<int8_t>(
reinterpret_cast<int8_t *>(host_data.data()),
@@ -612,6 +780,30 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
dist
);
break;
case library::NumericTypeID::kB1:
cutlass::reference::host::BlockFillRandom<uint1b_t>(
reinterpret_cast<uint1b_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU2:
cutlass::reference::host::BlockFillRandom<uint2b_t>(
reinterpret_cast<uint2b_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU4:
cutlass::reference::host::BlockFillRandom<uint4b_t>(
reinterpret_cast<uint4b_t *>(host_data.data()),
capacity_,
seed,
dist
);
break;
case library::NumericTypeID::kU8:
cutlass::reference::host::BlockFillRandom<uint8_t>(
reinterpret_cast<uint8_t *>(host_data.data()),
@@ -650,6 +842,67 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
copy_from_host(host_data.data());
}
void DeviceAllocation::initialize_random_sparsemeta_device(int seed, int MetaSizeInBits) {
if (!good()) {
throw std::runtime_error("Attempting to initialize invalid allocation.");
}
// Instantiate calls to CURAND here. This file takes a long time to compile for
// this reason.
switch (type_) {
case library::NumericTypeID::kU16:
cutlass::reference::device::BlockFillRandomSparseMeta<uint16_t>(
reinterpret_cast<uint16_t *>(pointer_),
capacity_,
seed,
MetaSizeInBits
);
break;
case library::NumericTypeID::kU32:
cutlass::reference::device::BlockFillRandomSparseMeta<uint32_t>(
reinterpret_cast<uint32_t *>(pointer_),
capacity_,
seed,
MetaSizeInBits
);
break;
default:
break;
}
}
void DeviceAllocation::initialize_random_sparsemeta_host(int seed, int MetaSizeInBits) {
if (!good()) {
throw std::runtime_error("Attempting to initialize invalid allocation.");
}
std::vector<uint8_t> host_data(bytes());
switch (type_) {
case library::NumericTypeID::kS16:
cutlass::reference::host::BlockFillRandomSparseMeta<uint16_t>(
reinterpret_cast<uint16_t *>(host_data.data()),
capacity_,
seed,
MetaSizeInBits
);
break;
case library::NumericTypeID::kS32:
cutlass::reference::host::BlockFillRandomSparseMeta<uint32_t>(
reinterpret_cast<uint32_t *>(host_data.data()),
capacity_,
seed,
MetaSizeInBits
);
break;
default:
break;
}
copy_from_host(host_data.data());
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns true if two blocks have exactly the same value
@@ -666,6 +919,18 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<half_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kBF16:
return reference::device::BlockCompareEqual<bfloat16_t>(
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kTF32:
return reference::device::BlockCompareEqual<tfloat32_t>(
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kF32:
return reference::device::BlockCompareEqual<float>(
reinterpret_cast<float const *>(ptr_A),
@@ -684,6 +949,18 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<complex<half_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCBF16:
return reference::device::BlockCompareEqual<complex<bfloat16_t>>(
reinterpret_cast<complex<bfloat16_t> const *>(ptr_A),
reinterpret_cast<complex<bfloat16_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCTF32:
return reference::device::BlockCompareEqual<complex<tfloat32_t>>(
reinterpret_cast<complex<tfloat32_t> const *>(ptr_A),
reinterpret_cast<complex<tfloat32_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kF64:
return reference::device::BlockCompareEqual<double>(
reinterpret_cast<double const *>(ptr_A),
@@ -695,6 +972,18 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<complex<double> const *>(ptr_A),
reinterpret_cast<complex<double> const *>(ptr_B),
capacity);
case library::NumericTypeID::kS2:
return reference::device::BlockCompareEqual<int2b_t>(
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS4:
return reference::device::BlockCompareEqual<int4b_t>(
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kS8:
return reference::device::BlockCompareEqual<int8_t>(
@@ -719,6 +1008,24 @@ bool DeviceAllocation::block_compare_equal(
reinterpret_cast<int64_t const *>(ptr_A),
reinterpret_cast<int64_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kB1:
return reference::device::BlockCompareEqual<uint1b_t>(
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU2:
return reference::device::BlockCompareEqual<uint2b_t>(
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU4:
return reference::device::BlockCompareEqual<uint4b_t>(
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_B),
capacity);
case library::NumericTypeID::kU8:
return reference::device::BlockCompareEqual<uint8_t>(
@@ -767,6 +1074,22 @@ bool DeviceAllocation::block_compare_relatively_equal(
static_cast<half_t>(epsilon),
static_cast<half_t>(nonzero_floor));
case library::NumericTypeID::kBF16:
return reference::device::BlockCompareRelativelyEqual<bfloat16_t>(
reinterpret_cast<bfloat16_t const *>(ptr_A),
reinterpret_cast<bfloat16_t const *>(ptr_B),
capacity,
static_cast<bfloat16_t>(epsilon),
static_cast<bfloat16_t>(nonzero_floor));
case library::NumericTypeID::kTF32:
return reference::device::BlockCompareRelativelyEqual<tfloat32_t>(
reinterpret_cast<tfloat32_t const *>(ptr_A),
reinterpret_cast<tfloat32_t const *>(ptr_B),
capacity,
static_cast<tfloat32_t>(epsilon),
static_cast<tfloat32_t>(nonzero_floor));
case library::NumericTypeID::kF32:
return reference::device::BlockCompareRelativelyEqual<float>(
reinterpret_cast<float const *>(ptr_A),
@@ -782,6 +1105,22 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<double>(epsilon),
static_cast<double>(nonzero_floor));
case library::NumericTypeID::kS2:
return reference::device::BlockCompareRelativelyEqual<int2b_t>(
reinterpret_cast<int2b_t const *>(ptr_A),
reinterpret_cast<int2b_t const *>(ptr_B),
capacity,
static_cast<int2b_t>(epsilon),
static_cast<int2b_t>(nonzero_floor));
case library::NumericTypeID::kS4:
return reference::device::BlockCompareRelativelyEqual<int4b_t>(
reinterpret_cast<int4b_t const *>(ptr_A),
reinterpret_cast<int4b_t const *>(ptr_B),
capacity,
static_cast<int4b_t>(epsilon),
static_cast<int4b_t>(nonzero_floor));
case library::NumericTypeID::kS8:
return reference::device::BlockCompareRelativelyEqual<int8_t>(
@@ -814,6 +1153,30 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity,
static_cast<int64_t>(epsilon),
static_cast<int64_t>(nonzero_floor));
case library::NumericTypeID::kB1:
return reference::device::BlockCompareRelativelyEqual<uint1b_t>(
reinterpret_cast<uint1b_t const *>(ptr_A),
reinterpret_cast<uint1b_t const *>(ptr_B),
capacity,
static_cast<uint1b_t>(epsilon),
static_cast<uint1b_t>(nonzero_floor));
case library::NumericTypeID::kU2:
return reference::device::BlockCompareRelativelyEqual<uint2b_t>(
reinterpret_cast<uint2b_t const *>(ptr_A),
reinterpret_cast<uint2b_t const *>(ptr_B),
capacity,
static_cast<uint2b_t>(epsilon),
static_cast<uint2b_t>(nonzero_floor));
case library::NumericTypeID::kU4:
return reference::device::BlockCompareRelativelyEqual<uint4b_t>(
reinterpret_cast<uint4b_t const *>(ptr_A),
reinterpret_cast<uint4b_t const *>(ptr_B),
capacity,
static_cast<uint4b_t>(epsilon),
static_cast<uint4b_t>(nonzero_floor));
case library::NumericTypeID::kU8:
return reference::device::BlockCompareRelativelyEqual<uint8_t>(
@@ -852,6 +1215,12 @@ bool DeviceAllocation::block_compare_relatively_equal(
// As a simplification, we can require bitwise equality. This avoids false positives.
// (i.e. "pass" really means passing. "Fail" may not actually mean failure given appropriate epsilon.)
//
case library::NumericTypeID::kCF16:
return reference::device::BlockCompareEqual<cutlass::complex<half_t> >(
reinterpret_cast<complex<half_t> const *>(ptr_A),
reinterpret_cast<complex<half_t> const *>(ptr_B),
capacity);
case library::NumericTypeID::kCF32:
return reference::device::BlockCompareEqual<cutlass::complex<float> >(
reinterpret_cast<complex<float> const *>(ptr_A),
@@ -865,7 +1234,9 @@ bool DeviceAllocation::block_compare_relatively_equal(
capacity);
default:
throw std::runtime_error("Unsupported numeric type");
{
throw std::runtime_error(std::string("Unsupported numeric type: ") + to_string(numeric_type));
}
}
}
@@ -928,13 +1299,17 @@ static void write_tensor_csv_static_tensor_view(
Layout layout(stride);
HostTensor<Element, Layout> host_tensor(extent, layout, false);
if (host_tensor.capacity() != allocation.capacity()) {
if (host_tensor.capacity() != allocation.batch_stride()) {
throw std::runtime_error("Unexpected capacity to equal.");
}
host_tensor.copy_in_device_to_host(static_cast<Element const *>(allocation.data()), host_tensor.capacity());
host_tensor.copy_in_device_to_host(
static_cast<Element const *>(allocation.data()),
allocation.batch_stride());
TensorViewWrite(out, host_tensor.host_view());
out << "\n\n";
}
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -951,9 +1326,42 @@ static void write_tensor_csv_static_type(
case library::LayoutTypeID::kColumnMajor:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajor>(out, allocation);
break;
case library::LayoutTypeID::kRowMajorInterleavedK2:
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<2>>(out, allocation);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK2:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<2>>(out, allocation);
break;
case library::LayoutTypeID::kRowMajorInterleavedK4:
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<4>>(out, allocation);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK4:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<4>>(out, allocation);
break;
case library::LayoutTypeID::kRowMajorInterleavedK16:
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<16>>(out, allocation);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK16:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<16>>(out, allocation);
break;
case library::LayoutTypeID::kRowMajorInterleavedK32:
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<32>>(out, allocation);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK32:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<32>>(out, allocation);
break;
case library::LayoutTypeID::kRowMajorInterleavedK64:
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<64>>(out, allocation);
break;
case library::LayoutTypeID::kColumnMajorInterleavedK64:
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<64>>(out, allocation);
break;
case library::LayoutTypeID::kTensorNHWC:
write_tensor_csv_static_tensor_view<T, layout::TensorNHWC>(out, allocation);
break;
case library::LayoutTypeID::kTensorNDHWC:
write_tensor_csv_static_tensor_view<T, layout::TensorNDHWC>(out, allocation);
break;
default:
throw std::runtime_error("Unhandled layout");
}
@@ -970,6 +1378,14 @@ void DeviceAllocation::write_tensor_csv(
write_tensor_csv_static_type<half_t>(out, *this);
break;
case library::NumericTypeID::kBF16:
write_tensor_csv_static_type<bfloat16_t>(out, *this);
break;
case library::NumericTypeID::kTF32:
write_tensor_csv_static_type<tfloat32_t>(out, *this);
break;
case library::NumericTypeID::kF32:
write_tensor_csv_static_type<float>(out, *this);
break;
@@ -977,6 +1393,14 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kF64:
write_tensor_csv_static_type<double>(out, *this);
break;
case library::NumericTypeID::kS2:
write_tensor_csv_static_type<int2b_t>(out, *this);
break;
case library::NumericTypeID::kS4:
write_tensor_csv_static_type<int4b_t>(out, *this);
break;
case library::NumericTypeID::kS8:
write_tensor_csv_static_type<int8_t>(out, *this);
@@ -993,6 +1417,18 @@ void DeviceAllocation::write_tensor_csv(
case library::NumericTypeID::kS64:
write_tensor_csv_static_type<int64_t>(out, *this);
break;
case library::NumericTypeID::kB1:
write_tensor_csv_static_type<uint1b_t>(out, *this);
break;
case library::NumericTypeID::kU2:
write_tensor_csv_static_type<uint2b_t>(out, *this);
break;
case library::NumericTypeID::kU4:
write_tensor_csv_static_type<uint4b_t>(out, *this);
break;
case library::NumericTypeID::kU8:
write_tensor_csv_static_type<uint8_t>(out, *this);
@@ -1010,6 +1446,10 @@ void DeviceAllocation::write_tensor_csv(
write_tensor_csv_static_type<uint64_t>(out, *this);
break;
case library::NumericTypeID::kCF16:
write_tensor_csv_static_type<cutlass::complex<half_t> >(out, *this);
break;
case library::NumericTypeID::kCF32:
write_tensor_csv_static_type<cutlass::complex<float> >(out, *this);
break;
+30 -4
View File
@@ -51,6 +51,9 @@ private:
/// Data type of contained elements
library::NumericTypeID type_;
/// Gets the stride between elements
size_t batch_stride_;
/// Capacity in elements of device allocation
size_t capacity_;
@@ -66,6 +69,9 @@ private:
/// Extent vector
std::vector<int> extent_;
/// Support allocating a 'batch' of non-overlapping tensors in contiguous memory
int batch_count_;
/// Buffer holding TensorRef instance to recently allocated memory
std::vector<uint8_t> tensor_ref_buffer_;
@@ -118,7 +124,8 @@ public:
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride = std::vector<int>());
std::vector<int> const &stride = std::vector<int>(),
int batch_count = 1);
~DeviceAllocation();
@@ -132,7 +139,8 @@ public:
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride = std::vector<int>());
std::vector<int> const &stride = std::vector<int>(),
int batch_count = 1);
/// Returns a buffer owning the tensor reference
std::vector<uint8_t> &tensor_ref() {
@@ -144,9 +152,12 @@ public:
/// Data type of contained elements
library::NumericTypeID type() const;
/// Pointer to device memory
/// Pointer to start of device memory allocation
void *data() const;
/// Pointer to the first element of a batch
void *batch_data(int batch_idx) const;
/// Gets the layout type
library::LayoutTypeID layout() const;
@@ -156,6 +167,15 @@ public:
/// Gets the extent vector
std::vector<int> const & extent() const;
/// Gets the number of adjacent tensors in memory
int batch_count() const;
/// Gets the stride (in units of elements) beteween items
int64_t batch_stride() const;
/// Gets the stride (in units of bytes) beteween items
int64_t batch_stride_bytes() const;
/// Capacity of allocation in number of elements
size_t capacity() const;
@@ -165,9 +185,15 @@ public:
/// Initializes a device allocation to a random distribution using cuRAND
void initialize_random_device(int seed, Distribution dist);
/// Initializes a device allocation to a random distribution using cuRAND
/// Initializes a host allocation to a random distribution using std::cout
void initialize_random_host(int seed, Distribution dist);
/// Initializes a device allocation to a random distribution using cuRAND
void initialize_random_sparsemeta_device(int seed, int MetaSizeInBits);
/// Initializes a host allocation to a random distribution using std::cout
void initialize_random_sparsemeta_host(int seed, int MetaSizeInBits);
/// Copies from an equivalent-sized tensor in device memory
void copy_from_device(void const *ptr);
+55 -9
View File
@@ -52,9 +52,10 @@ DeviceAllocation *DeviceContext::allocate_tensor(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride) {
std::vector<int> const &stride,
int batch_count) {
device_memory_.emplace_back(type, layout_id, extent, stride);
device_memory_.emplace_back(type, layout_id, extent, stride, batch_count);
DeviceAllocation *allocation = &device_memory_.back();
allocations_[name] = allocation;
@@ -68,10 +69,11 @@ DeviceAllocation *DeviceContext::allocate_tensor(
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride) {
std::vector<int> const &stride,
int batch_count) {
DeviceAllocation *allocation =
allocate_tensor(name, type, layout_id, extent, stride);
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
if (options.initialization.enabled) {
Distribution data_distribution = options.initialization.data_distribution;
@@ -81,11 +83,23 @@ DeviceAllocation *DeviceContext::allocate_tensor(
// change data distribution based on bit width
switch(type) {
case library::NumericTypeID::kB1:
data_distribution.set_uniform(0, 2, 0);
break;
case library::NumericTypeID::kS8:
data_distribution.set_uniform(0, 1, 0);
break;
case library::NumericTypeID::kS2:
data_distribution.set_uniform(-1, 1, 0);
break;
case library::NumericTypeID::kS4:
data_distribution.set_uniform(-2, 2, 0);
break;
case library::NumericTypeID::kU2:
data_distribution.set_uniform(0, 2, 0);
break;
case library::NumericTypeID::kU4:
data_distribution.set_uniform(0, 2, 0);
break;
case library::NumericTypeID::kS8:
data_distribution.set_uniform(-3, 3, 0);
break;
case library::NumericTypeID::kU8:
data_distribution.set_uniform(0, 4, 0);
break;
@@ -96,18 +110,50 @@ DeviceAllocation *DeviceContext::allocate_tensor(
if (options.initialization.provider == library::Provider::kReferenceDevice) {
allocation->initialize_random_device(
options.initialization.seed,
data_distribution);
data_distribution);
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
allocation->initialize_random_host(
options.initialization.seed,
data_distribution);
data_distribution);
}
}
return allocation;
}
/// Allocates memory for sparse meta data
DeviceAllocation *DeviceContext::allocate_sparsemeta_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
library::NumericTypeID type_a,
std::vector<int> const &extent,
std::vector<int> const &stride,
int batch_count) {
DeviceAllocation *allocation =
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
if (options.initialization.enabled) {
// TF32 has 4bit meta data. The rest has 2bit.
int MetaSizeInBits = (cutlass::library::sizeof_bits(type_a) == 32) ? 4 : 2;
if (options.initialization.provider == library::Provider::kReferenceDevice) {
allocation->initialize_random_sparsemeta_device(
options.initialization.seed,
MetaSizeInBits);
}
else if (options.initialization.provider == library::Provider::kReferenceHost) {
allocation->initialize_random_sparsemeta_host(
options.initialization.seed,
MetaSizeInBits);
}
}
return allocation;
}
/// Clears named allocations (but does not necessarily free memory)
void DeviceContext::clear() {
allocations_.clear();
+16 -2
View File
@@ -33,6 +33,7 @@
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "options.h"
#include "device_allocation.h"
@@ -76,7 +77,8 @@ public:
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride = std::vector<int>());
std::vector<int> const &stride = std::vector<int>(),
int batch_count = 1);
/// Allocates memory of a given type, capacity (elements), and name
DeviceAllocation *allocate_tensor(
@@ -85,7 +87,19 @@ public:
library::NumericTypeID type,
library::LayoutTypeID layout_id,
std::vector<int> const &extent,
std::vector<int> const &stride = std::vector<int>());
std::vector<int> const &stride = std::vector<int>(),
int batch_count = 1);
/// Allocates memory for sparse meta data
DeviceAllocation *allocate_sparsemeta_tensor(
Options const &options,
std::string const &name,
library::NumericTypeID type,
library::LayoutTypeID layout_id,
library::NumericTypeID type_a,
std::vector<int> const &extent,
std::vector<int> const &stride = std::vector<int>(),
int batch_count = 1);
/// Clears named allocations (but does not necessarily free memory)
void clear();
+347 -34
View File
@@ -37,6 +37,9 @@
#include "gemm_operation_profiler.h"
#include "gpu_timer.h"
#include "cutlass/library/library.h"
#include "cutlass/library/handle.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
@@ -135,6 +138,8 @@ Status GemmOperationProfiler::GemmProblem::parse(
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
this->mode = library::GemmUniversalMode::kGemm;
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
@@ -151,6 +156,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
this->mode = library::GemmUniversalMode::kGemm;
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
@@ -160,6 +166,14 @@ Status GemmOperationProfiler::GemmProblem::parse(
// default value
this->batch_count = 1;
}
else if (this->batch_count > 1) {
this->mode = library::GemmUniversalMode::kBatched;
}
if (this->split_k_slices > 1 && this->batch_count > 1) {
// At least one of these must be one
return Status::kErrorInvalidProblem;
}
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
return Status::kErrorInvalidProblem;
@@ -209,6 +223,48 @@ Status GemmOperationProfiler::GemmProblem::parse(
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
}
bytes *= batch_count;
return bytes;
}
/// Total number of flops computed
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
case library::MathOperationID::kMultiplyAddComplex:
flops_ *= 4;
break;
case library::MathOperationID::kMultiplyAddGaussianComplex:
flops_ *= 3;
break;
default: break;
}
return flops_;
}
/// Initializes a performance result
void GemmOperationProfiler::GemmProblem::initialize_result(
PerformanceResult &result,
@@ -266,6 +322,7 @@ Status GemmOperationProfiler::initialize_configuration(
return status;
}
gemm_workspace_.configuration.mode = problem_.mode;
gemm_workspace_.configuration.problem_size.m() = int(problem_.m);
gemm_workspace_.configuration.problem_size.n() = int(problem_.n);
gemm_workspace_.configuration.problem_size.k() = int(problem_.k);
@@ -273,8 +330,13 @@ Status GemmOperationProfiler::initialize_configuration(
gemm_workspace_.configuration.ldb = problem_.ldb;
gemm_workspace_.configuration.ldc = problem_.ldc;
gemm_workspace_.configuration.ldd = problem_.ldc;
//gemm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
gemm_workspace_.configuration.batch_count = int(problem_.split_k_slices);
if (problem_.mode == library::GemmUniversalMode::kBatched) {
gemm_workspace_.configuration.batch_count = problem_.batch_count;
}
else {
gemm_workspace_.configuration.batch_count = problem_.split_k_slices;
}
gemm_workspace_.arguments.A = nullptr;
gemm_workspace_.arguments.B = nullptr;
@@ -305,32 +367,10 @@ void GemmOperationProfiler::initialize_result_(
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
// Input bytes read and Output bytes written for the gemm problem
result.bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) * problem_.k +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.n / 8) * problem_.k +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(problem_.beta.begin(), problem_.beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
result.bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
}
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
result.bytes = problem_.bytes(operation_desc);
result.flops = problem_.flops(operation_desc);
result.runtime = 0;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
case library::MathOperationID::kMultiplyAddComplex:
result.flops *= 4;
break;
default: break;
}
}
/// Initializes workspace
@@ -345,6 +385,21 @@ Status GemmOperationProfiler::initialize_workspace(
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
// Compute the number of copies of the problem to avoid L2 camping.
if (!options.profiling.workspace_count) {
int64_t bytes = problem_.bytes(operation_desc);
if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
gemm_workspace_.problem_count =
1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
}
else {
gemm_workspace_.problem_count = 1;
}
}
else {
gemm_workspace_.problem_count = options.profiling.workspace_count;
}
if (options.execution_mode != ExecutionMode::kDryRun) {
gemm_workspace_.A = device_context.allocate_tensor(
@@ -353,7 +408,8 @@ Status GemmOperationProfiler::initialize_workspace(
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)}
{int(problem_.lda)},
problem_.batch_count * gemm_workspace_.problem_count
);
gemm_workspace_.B = device_context.allocate_tensor(
@@ -362,7 +418,8 @@ Status GemmOperationProfiler::initialize_workspace(
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)}
{int(problem_.ldb)},
problem_.batch_count * gemm_workspace_.problem_count
);
gemm_workspace_.C = device_context.allocate_tensor(
@@ -371,7 +428,8 @@ Status GemmOperationProfiler::initialize_workspace(
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
);
gemm_workspace_.Computed = device_context.allocate_tensor(
@@ -379,7 +437,8 @@ Status GemmOperationProfiler::initialize_workspace(
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
);
gemm_workspace_.Reference = device_context.allocate_tensor(
@@ -387,7 +446,8 @@ Status GemmOperationProfiler::initialize_workspace(
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
{int(problem_.ldc)},
problem_.batch_count * gemm_workspace_.problem_count
);
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
@@ -458,6 +518,10 @@ bool GemmOperationProfiler::verify_cutlass(
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
//
// Run the CUTLASS operation
@@ -512,6 +576,8 @@ bool GemmOperationProfiler::verify_cutlass(
}
}
#endif // #if CUTLASS_ENABLE_CUBLAS
verify_with_reference_(options, report, device_context, operation, problem_space, problem);
// Update disposition to worst case verification outcome among all
// verification providers which are supported
@@ -561,7 +627,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
if (status != CUBLAS_STATUS_SUCCESS) {
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
return true;
}
@@ -589,9 +655,13 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.C = gemm_workspace_.Reference->data();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Reference->batch_stride();
gemm_workspace_.arguments.D = gemm_workspace_.Reference->data();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Reference->batch_stride();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
@@ -615,7 +685,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
// Handle errors
if (status != CUBLAS_STATUS_SUCCESS) {
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
return true;
}
@@ -626,7 +696,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
options,
*gemm_workspace_.Computed,
*gemm_workspace_.Reference
*gemm_workspace_.Reference,
gemm_workspace_.Computed->batch_stride()
);
// Save workspace if incorrect
@@ -653,6 +724,150 @@ bool GemmOperationProfiler::verify_with_cublas_(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Verifies CUTLASS against host and device references
bool GemmOperationProfiler::verify_with_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::GemmDescription const &gemm_desc =
static_cast<library::GemmDescription const &>(operation->description());
//
// Initialize state
//
library::Provider references[] = {
library::Provider::kReferenceDevice,
library::Provider::kReferenceHost
};
for (auto provider : references) {
// Skip providers that are not enabled
if (!options.verification.provider_enabled(provider)) {
continue;
}
void *ptr_A = gemm_workspace_.A->data();
void *ptr_B = gemm_workspace_.B->data();
void *ptr_C = gemm_workspace_.C->data();
void *ptr_D = gemm_workspace_.Reference->data();
// To support the host-side reference, conditionally allocate and
// copy tensors to host memory.
std::vector<uint8_t> host_data_A;
std::vector<uint8_t> host_data_B;
std::vector<uint8_t> host_data_C;
std::vector<uint8_t> host_data_D;
if (provider == library::Provider::kReferenceHost) {
host_data_A.resize(gemm_workspace_.A->bytes());
ptr_A = host_data_A.data();
gemm_workspace_.A->copy_to_host(ptr_A);
host_data_B.resize(gemm_workspace_.B->bytes());
ptr_B = host_data_B.data();
gemm_workspace_.B->copy_to_host(ptr_B);
host_data_C.resize(gemm_workspace_.C->bytes());
ptr_C = host_data_C.data();
gemm_workspace_.C->copy_to_host(ptr_C);
host_data_D.resize(gemm_workspace_.Reference->bytes());
ptr_D = host_data_D.data();
}
//
// Launch
//
library::Handle handle;
handle.set_provider(provider);
Status status = handle.gemm_universal(
library::GemmUniversalMode::kGemm,
gemm_workspace_.configuration.problem_size.m(),
gemm_workspace_.configuration.problem_size.n(),
gemm_workspace_.configuration.problem_size.k(),
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
problem_.alpha.data(),
gemm_desc.A.element,
gemm_desc.A.layout,
gemm_desc.transform_A,
ptr_A,
int(gemm_workspace_.configuration.lda),
gemm_desc.B.element,
gemm_desc.B.layout,
gemm_desc.transform_B,
ptr_B,
int(gemm_workspace_.configuration.ldb),
problem_.beta.data(),
gemm_desc.C.element,
ptr_C,
int(gemm_workspace_.configuration.ldc),
ptr_D,
int(gemm_workspace_.configuration.ldd),
gemm_workspace_.configuration.batch_count,
gemm_workspace_.A->batch_stride(),
gemm_workspace_.B->batch_stride(),
gemm_workspace_.C->batch_stride(),
gemm_workspace_.Reference->batch_stride()
);
if (status != Status::kSuccess) {
results_.back().verification_map[provider] = Disposition::kNotRun;
return true;
}
results_.back().status = status;
if (provider == library::Provider::kReferenceHost) {
gemm_workspace_.Reference->copy_from_host(ptr_D);
}
//
// Verify results
//
results_.back().verification_map[provider] = compare_tensors(
options,
*gemm_workspace_.Computed,
*gemm_workspace_.Reference,
gemm_workspace_.Computed->batch_stride()
);
// Save workspace if incorrect
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
results_.back().verification_map[provider] == Disposition::kIncorrect) {
save_workspace(
device_context,
options,
gemm_desc,
library::Provider::kCUTLASS,
provider);
}
}
return true;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Measures performance results
bool GemmOperationProfiler::profile(
Options const &options,
@@ -672,6 +887,10 @@ bool GemmOperationProfiler::profile(
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
results_.back().status = profile_cutlass_(
results_.back().runtime,
@@ -687,6 +906,100 @@ bool GemmOperationProfiler::profile(
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Method to profile a CUTLASS Operation
Status GemmOperationProfiler::profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace) {
GpuTimer timer;
//
// Optional sleep to limit power consumption and thermals
//
sleep(options.profiling.sleep_duration);
//
// Warmup loop
//
Status status;
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
int problem_idx = (iteration % gemm_workspace_.problem_count) * problem_.batch_count;
gemm_workspace_.arguments.A = gemm_workspace_.A->batch_data(problem_idx);
gemm_workspace_.arguments.B = gemm_workspace_.B->batch_data(problem_idx);
gemm_workspace_.arguments.C = gemm_workspace_.C->batch_data(problem_idx);
gemm_workspace_.arguments.D = gemm_workspace_.Computed->batch_data(problem_idx);
// Execute the CUTLASS operation
status = operation->run(
&gemm_workspace_.arguments,
host_workspace,
device_workspace);
if (status != Status::kSuccess) {
return status;
}
}
//
// Initialize GPU timer
//
timer.start();
//
// Profiling loop
//
int Iterations = options.profiling.iterations;
int iteration = 0;
for (; iteration < Iterations; ++iteration) {
// Iterate over copies of the problem in memory
int workspace_idx = options.profiling.warmup_iterations + iteration;
int problem_idx = (workspace_idx % gemm_workspace_.problem_count) * problem_.batch_count;
gemm_workspace_.arguments.A = gemm_workspace_.A->batch_data(problem_idx);
gemm_workspace_.arguments.B = gemm_workspace_.B->batch_data(problem_idx);
gemm_workspace_.arguments.C = gemm_workspace_.C->batch_data(problem_idx);
gemm_workspace_.arguments.D = gemm_workspace_.Computed->batch_data(problem_idx);
status = operation->run(
arguments,
host_workspace,
device_workspace);
if (status != Status::kSuccess) {
return status;
}
}
//
// Wait for completion
//
timer.stop_and_wait();
//
// Update performance result
//
runtime = timer.duration(iteration);
return status;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
+34 -3
View File
@@ -59,6 +59,8 @@ public:
/// Problem structure obtained from problem space
struct GemmProblem {
cutlass::library::GemmUniversalMode mode;
int64_t m;
int64_t n;
int64_t k;
@@ -67,14 +69,15 @@ public:
int64_t ldc;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
int split_k_slices;
int batch_count;
//
// Methods
//
GemmProblem():
mode(library::GemmUniversalMode::kGemm),
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), split_k_slices(1), batch_count(1) { }
/// Parses the problem
@@ -83,6 +86,12 @@ public:
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Total number of bytes loaded
int64_t bytes(library::GemmDescription const &operation_desc) const;
/// Total number of flops computed
int64_t flops(library::GemmDescription const &operation_desc) const;
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
@@ -99,6 +108,10 @@ public:
DeviceAllocation *Computed;
DeviceAllocation *Reference;
/// Number of copies of the problem workspace which are visited sequentially during
/// profiling to avoid camping in the last level cache.
int problem_count;
library::GemmUniversalConfiguration configuration;
library::GemmUniversalArguments arguments;
@@ -113,7 +126,7 @@ public:
//
GemmWorkspace():
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr), problem_count(1) { }
};
protected:
@@ -200,6 +213,24 @@ protected:
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against host and device references
bool verify_with_reference_(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Method to profile a CUTLASS Operation
Status profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void *arguments,
void *host_workspace,
void *device_workspace);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
+17 -4
View File
@@ -268,6 +268,7 @@ int OperationProfiler::profile_all(
std::string operation_name(operation->description().name);
// Filter kernels by name
bool filtered_by_name = options.operation_names.empty();
if (!filtered_by_name) {
@@ -279,6 +280,13 @@ int OperationProfiler::profile_all(
}
}
for (auto const & op_name : options.excluded_operation_names) {
if (find_string_matches_(op_name, operation_name)) {
filtered_by_name = false;
break;
}
}
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
continue;
}
@@ -410,7 +418,8 @@ void OperationProfiler::sleep(int sleep_duration) {
Disposition OperationProfiler::compare_tensors(
Options const &options,
DeviceAllocation &experimental,
DeviceAllocation &reference) {
DeviceAllocation &reference,
int64_t count) {
if (experimental.type() != reference.type()) {
return Disposition::kIncorrect;
@@ -418,6 +427,10 @@ Disposition OperationProfiler::compare_tensors(
bool passed = false;
if (count == 0) {
count = reference.capacity();
}
if (options.verification.epsilon == 0) {
// bit-level equality
@@ -425,7 +438,7 @@ Disposition OperationProfiler::compare_tensors(
experimental.type(),
experimental.data(),
reference.data(),
experimental.capacity());
count);
}
else {
@@ -434,7 +447,7 @@ Disposition OperationProfiler::compare_tensors(
experimental.type(),
experimental.data(),
reference.data(),
experimental.capacity(),
count,
options.verification.epsilon,
options.verification.nonzero_floor);
}
@@ -483,7 +496,7 @@ Status OperationProfiler::profile_cutlass_(
double &runtime,
Options const &options,
library::Operation const *operation,
void const *arguments,
void *arguments,
void *host_workspace,
void *device_workspace) {
+3 -2
View File
@@ -189,7 +189,8 @@ public:
static Disposition compare_tensors(
Options const &options,
DeviceAllocation &experimental,
DeviceAllocation &reference);
DeviceAllocation &reference,
int64_t count = 0);
static void save_workspace(
DeviceContext &device_context,
@@ -225,7 +226,7 @@ protected:
double &runtime,
Options const &options,
library::Operation const *operation,
void const *arguments,
void *arguments,
void *host_workspace,
void *device_workspace);
+31 -2
View File
@@ -70,6 +70,16 @@ Options::Device::Device(cutlass::CommandLine const &cmdline) {
properties.major = cc / 10;
properties.minor = cc % 10;
}
// Permit overriding the L2 cache capacity
if (cmdline.check_cmd_line_flag("llc-capacity")) {
int llc_capacity = 0;
cmdline.get_cmd_line_argument("llc-capacity", llc_capacity, 0);
if (llc_capacity >= 0) {
properties.l2CacheSize = (llc_capacity << 10);
}
}
}
@@ -107,7 +117,12 @@ void Options::Device::print_usage(std::ostream &out) const {
out
<< " --compute-capability=<int> "
<< " Override the compute capability.\n\n";
<< " Override the compute capability.\n\n"
<< " --llc-capacity=<capacity in KiB> "
<< " Capacity of last-level cache in kilobytes. If this is non-zero," << end_of_line
<< " profiling phases cycle through different input tensors to induce" << end_of_line
<< " capacity misses in the L2.\n\n";
}
@@ -189,6 +204,7 @@ Options::Initialization::Initialization(cutlass::CommandLine const &cmdline) {
// set uniform data distribution with range [-4, 4]
data_distribution.set_uniform(-4, 4, 0);
}
}
@@ -364,7 +380,8 @@ void Options::Library::print_options(std::ostream &out, int indent) const {
/////////////////////////////////////////////////////////////////////////////////////////////////
Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("workspace-count", workspace_count, 0);
cmdline.get_cmd_line_argument("warmup-iterations", warmup_iterations, 10);
cmdline.get_cmd_line_argument("profiling-iterations", iterations, 100);
cmdline.get_cmd_line_argument("sleep-duration", sleep_duration, 50);
@@ -391,6 +408,11 @@ void Options::Profiling::print_usage(std::ostream &out) const {
out << "Profiling:\n"
<< " --workspace-count=<workspace count> "
<< " Number of discrete workspaces maintained to avoid cache-resident " << end_of_line
<< " If zero (default), the amount is chosen for each workload based on " << end_of_line
<< " capacity of the last-level cache.\n\n"
<< " --profiling-iterations=<iterations> "
<< " Number of iterations to profile each kernel. If zero, kernels" << end_of_line
<< " are launched up to the profiling duration.\n\n"
@@ -672,6 +694,10 @@ Options::Options(cutlass::CommandLine const &cmdline):
cmdline.get_cmd_line_arguments("kernels", operation_names);
}
if (cmdline.check_cmd_line_flag("ignore-kernels")) {
cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
}
// Prevent launches on the device for anything other than CUTLASS operation
if (execution_mode == ExecutionMode::kTrace) {
initialization.provider = library::Provider::kReferenceHost;
@@ -706,6 +732,9 @@ void Options::print_usage(std::ostream &out) const {
<< " Filter operations by kernel names. For example, call all kernels with" << end_of_line
<< " (\"s1688\" and \"nt\") or (\"s844\" and \"tn\" and \"align8\") in their" << end_of_line
<< " operation name using --kernels=\"s1688*nt, s884*tn*align8\"\n\n"
<< " --ignore-kernels=<string_list> "
<< " Excludes kernels whose names match anything in this list.\n\n"
;
//
+7
View File
@@ -175,6 +175,9 @@ public:
/// Options related to profiling
struct Profiling {
/// Number of workspaces to rotate through to avoid cache-resident working sets
int workspace_count;
/// Number of iterations to warmup each kernel prior to profiling
int warmup_iterations;
@@ -273,6 +276,10 @@ public:
/// Vector of operation name substrings
std::vector<std::string> operation_names;
/// Vector of operation name substrings
std::vector<std::string> excluded_operation_names;
//
// Detailed configuration options
@@ -263,6 +263,10 @@ std::ostream & PerformanceReport::print_csv_header_(
<< ",OperationKind,Operation,Disposition,Status";
for (auto const &arg_name : argument_names_) {
// Operand E is internal to the sparse kernel
if (arg_name.compare("E") == 0)
continue;
out << "," << arg_name;
}
+77 -7
View File
@@ -577,19 +577,65 @@ void ProblemSpace::parse_(KernelArgument *arg, CommandLine const &cmdline) {
std::vector<std::vector<std::string> > tokens;
cmdline.get_cmd_line_argument_ranges(alias.c_str(), tokens);
for (auto const &range_tokens : tokens) {
for (auto &range_tokens : tokens) {
if (!range_tokens.empty()) {
Range range(lexical_cast<int64_t>(range_tokens.front()));
if (range_tokens.size() > 1) {
range.last = lexical_cast<int64_t>(range_tokens.at(1));
Range range;
if (range_tokens.front() == "rand") {
range.mode = Range::Mode::kRandom;
}
else if (range_tokens.front() == "randlg2") {
range.mode = Range::Mode::kRandomLog2;
}
if (range_tokens.size() > 2) {
range.increment = lexical_cast<int64_t>(range_tokens.at(2));
}
switch (range.mode) {
case Range::Mode::kSequence:
{
range.first = lexical_cast<int64_t>(range_tokens.front());
if (range_tokens.size() > 1) {
range.last = lexical_cast<int64_t>(range_tokens.at(1));
}
else {
range.last = range.first;
}
if (range_tokens.size() > 2) {
range.increment = lexical_cast<int64_t>(range_tokens.at(2));
}
else {
range.increment = 1;
}
}
break;
case Range::Mode::kRandom: // fall-through
case Range::Mode::kRandomLog2:
{
if (range_tokens.size() < 4) {
throw std::runtime_error(
"Range of mode 'rand' must have four tokens showing "
"the minimum, maximum, and number of iterations. For example, "
"rand:16:128:1000");
}
range.minimum = lexical_cast<int64_t>(range_tokens.at(1));
range.maximum = lexical_cast<int64_t>(range_tokens.at(2));
range.first = 1;
range.last = lexical_cast<int64_t>(range_tokens.at(3));
range.increment = 1;
if (range_tokens.size() > 4) {
range.divisible = lexical_cast<int64_t>(range_tokens.at(4));
}
}
break;
default:
throw std::runtime_error("Unsupported range mode.");
break;
}
integer->ranges.push_back(range);
}
}
@@ -713,6 +759,30 @@ bool arg_as_int(int64_t &int_value, KernelArgument::Value const *value_ptr) {
return false;
}
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_int(int &int_value, KernelArgument::Value const *value_ptr) {
int64_t value64;
bool obtained = arg_as_int(value64, value_ptr);
if (obtained) {
int_value = int(value64);
return true;
}
return false;
}
/// Lexically casts an argument to an int
bool arg_as_int(
int &int_value,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
size_t idx = problem_space.argument_index(name);
KernelArgument::Value const *value_ptr = problem.at(idx).get();
return arg_as_int(int_value, value_ptr);
}
/// Lexically casts an argument to an int64
bool arg_as_int(
int64_t &int_value,
+94 -7
View File
@@ -49,6 +49,7 @@
#include <vector>
#include <memory>
#include <unordered_map>
#include <cstdlib>
// CUTLASS Utility includes
#include "cutlass/util/command_line.h"
@@ -307,10 +308,18 @@ struct Range {
// Type definitions
//
enum class Mode {
kSequence,
kRandom,
kRandomLog2,
kInvalid
};
struct Iterator {
int64_t value;
int64_t increment;
Range const *range;
//
// Methods
@@ -318,9 +327,10 @@ struct Range {
Iterator(
int64_t value_ = 0,
int64_t increment_ = 1
int64_t increment_ = 1,
Range const *range_ = nullptr
):
value(value_), increment(increment_) { }
value(value_), increment(increment_), range(range_) { }
Iterator & operator++() {
value += increment;
@@ -341,7 +351,50 @@ struct Range {
return !(*this == it);
}
static int64_t round(int64_t value, int64_t divisible) {
int64_t rem = (value % divisible);
// Round either up or down
if (rem > divisible / 2) {
value += (divisible - rem);
}
else {
value -= rem;
}
return value;
}
int64_t at() const {
if (!range) {
return value;
}
switch (range->mode) {
case Mode::kSequence: return value;
case Mode::kRandom: {
double rnd = double(range->minimum) +
double(std::rand()) / double(RAND_MAX) * (double(range->maximum) - double(range->minimum));
int64_t value = int64_t(rnd);
return round(value, range->divisible);
}
break;
case Mode::kRandomLog2: {
double lg2_minimum = std::log(double(range->minimum)) / std::log(2.0);
double lg2_maximum = std::log(double(range->maximum)) / std::log(2.0);
double rnd = lg2_minimum + double(std::rand()) / double(RAND_MAX) * (lg2_maximum - lg2_minimum);
int64_t value = int64_t(std::pow(2.0, rnd));
return round(value, range->divisible);
}
break;
default: break;
}
return value;
}
@@ -357,20 +410,29 @@ struct Range {
int64_t first; ///< first element in range
int64_t last; ///< last element in range
int64_t increment; ///< additive increment between values
Mode mode; ///< mode selection enables alternative values
int64_t minimum; ///< minimum value to return
int64_t maximum; ///< maximum value to return
int64_t divisible; ///< rounds value down to an integer multiple of this value
//
// Methods
//
/// Default constructor - range acts as a scalar
Range(int64_t first_ = 0): first(first_), last(first_), increment(1) { }
Range(int64_t first_ = 0): first(first_), last(first_), increment(1), mode(Mode::kSequence), minimum(0), maximum(0), divisible(1) { }
/// Range acts as a range
Range(
int64_t first_,
int64_t last_,
int64_t increment_ = 1
): first(first_), last(last_), increment(increment_) {
int64_t increment_ = 1,
Mode mode_ = Mode::kSequence,
int64_t minimum_ = 0,
int64_t maximum_ = 0,
int64_t divisible_ = 1
): first(first_), last(last_), increment(increment_), mode(mode_), minimum(minimum_), maximum(maximum_), divisible(divisible_) {
// Helpers to avoid constructing invalid ranges
if (increment > 0) {
@@ -389,14 +451,29 @@ struct Range {
}
}
/// Helper to construct a sequence range
static Range Sequence(int64_t first_, int64_t last_, int64_t increment_ = 1) {
return Range(first_, last_, increment_, Mode::kSequence);
}
/// Helper to construct a range that is a random distribution
static Range Random(int64_t minimum_, int64_t maximum_, int64_t count_, int64_t divisible_ = 1) {
return Range(1, count_, 1, Mode::kRandom, minimum_, maximum_, divisible_);
}
/// Helper to construct a range that is a random distribution over a log scale
static Range RandomLog2(int64_t minimum_, int64_t maximum_, int64_t count_, int64_t divisible_ = 1) {
return Range(1, count_, 1, Mode::kRandomLog2, minimum_, maximum_, divisible_);
}
/// Returns an iterator to the first element within the range
Iterator begin() const {
return Iterator(first, increment);
return Iterator(first, increment, this);
}
/// Returns an iterator to the first element *after* the range
Iterator end() const {
return Iterator(first + ((last - first)/increment + 1) * increment, increment);
return Iterator(first + ((last - first)/increment + 1) * increment, increment, this);
}
};
@@ -770,9 +847,19 @@ private:
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Lexically casts an argument to an int if it is defined. Returns true if not null.
bool arg_as_int(int &int_value, KernelArgument::Value const *value_ptr);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_int(int64_t &int_value, KernelArgument::Value const *value_ptr);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_int(
int &int_value,
char const *name,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
bool arg_as_int(
int64_t &int_value,
@@ -0,0 +1,560 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief Execution environment
*/
#include <iostream>
#include <stdexcept>
#include <iomanip>
#include <ios>
#include "cublas_helpers.h"
#include "sparse_gemm_operation_profiler.h"
#include "gpu_timer.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
SparseGemmOperationProfiler::SparseGemmOperationProfiler(Options const &options):
OperationProfiler(
options,
library::OperationKind::kSparseGemm,
{
{ArgumentTypeID::kEnumerated, {"gemm_kind"}, "Variant of GEMM (e.g. gemm, planar complex, batched, ...)"},
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
{ArgumentTypeID::kTensor, {"E"}, "Tensor storing the E operand"},
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
{ArgumentTypeID::kInteger, {"split_k_slices"}, "Number of partitions of K dimension"},
{ArgumentTypeID::kInteger, {"batch_count"}, "Number of GEMMs computed in one batch"},
}
) {
description_ = " Structured sparse GEMM. D = alpha * A*B + beta * C";
}
/// Destructor
SparseGemmOperationProfiler::~SparseGemmOperationProfiler() {
}
/// Prints usage statement for the math function
void SparseGemmOperationProfiler::print_usage(std::ostream &out) const {
out << "Sparse GEMM" << "\n\n";
OperationProfiler::print_usage(out);
}
/// Prints examples
void SparseGemmOperationProfiler::print_examples(std::ostream &out) const {
out << "\nExamples:\n\n"
<< "Profile a particular problem size:\n"
<< " $ cutlass_profiler --operation=SparseGemm --m=1024 --n=1024 --k=128\n\n"
<< "Schmoo over problem size and beta:\n"
<< " $ cutlass_profiler --operation=SparseGemm --m=1024:4096:256 --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
<< "Schmoo over accumulator types:\n"
<< " $ cutlass_profiler --operation=SparseGemm --accumulator-type=f16,f32\n\n"
<< "Run when A is f16 with column-major and B is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
<< " $ cutlass_profiler --operation=SparseGemm --A=f16:column --B=*:row\n\n"
<< "Using various input value distribution:\n"
<< " $ cutlass_profiler --operation=SparseGemm --dist=uniform,min:0,max:3\n"
<< " $ cutlass_profiler --operation=SparseGemm --dist=gaussian,mean:0,stddev:3\n"
<< " $ cutlass_profiler --operation=SparseGemm --dist=sequential,start:0,delta:1\n\n"
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
<< " $ cutlass_profiler --operation=SparseGemm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
<< "Test your changes to gemm kernels with a quick functional test and save results in functional-test.csv:\n"
<< " $ cutlass_profiler --operation=SparseGemm \\ \n"
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
<< " --providers=cutlass --output=functional-test.csv\n\n";
}
/////////////////////////////////////////////////////////////////////////////////////////////////
Status SparseGemmOperationProfiler::SparseGemmProblem::parse(
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!arg_as_int(this->m, "m", problem_space, problem)) {
// default value
this->m = 1024;
}
if (!arg_as_int(this->n, "n", problem_space, problem)) {
// default value
this->n = 1024;
}
if (!arg_as_int(this->k, "k", problem_space, problem)) {
// default value
this->k = 1024;
}
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
// default value
this->split_k_slices = 1;
}
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
// default value
this->batch_count = 1;
}
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
return Status::kErrorInvalidProblem;
}
if (!tensor_description_satisfies(operation_desc.B, "B", problem_space, problem)) {
return Status::kErrorInvalidProblem;
}
if (!tensor_description_satisfies(operation_desc.C, "C", problem_space, problem)) {
return Status::kErrorInvalidProblem;
}
if (!tensor_description_satisfies(operation_desc.E, "E", problem_space, problem)) {
return Status::kErrorInvalidProblem;
}
if (!arg_as_scalar(
this->alpha,
operation_desc.element_epilogue,
"alpha",
problem_space,
problem)) {
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
return Status::kErrorInternal;
}
}
if (!arg_as_scalar(
this->beta,
operation_desc.element_epilogue,
"beta",
problem_space,
problem)) {
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
return Status::kErrorInternal;
}
}
this->elements_per_128b =
128 / library::sizeof_bits(operation_desc.A.element);
this->lda = DeviceAllocation::get_packed_layout(
operation_desc.A.layout,
{int(this->m), int(this->k) / int(this->sparse)})
.front();
this->ldb = DeviceAllocation::get_packed_layout(
operation_desc.B.layout, {int(this->k), int(this->n)}).front();
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
this->lde =
DeviceAllocation::get_packed_layout(
operation_desc.E.layout,
{int(this->m), int(this->k / this->sparse / this->elements_per_128b)})
.front();
return Status::kSuccess;
}
/// Initializes a performance result
void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
PerformanceResult &result,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space) {
result.arguments.resize(problem_space.rank());
set_argument(result, "gemm_kind", problem_space, library::to_string(operation_desc.gemm_kind));
set_argument(result, "A", problem_space,
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
set_argument(result, "B", problem_space,
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
set_argument(result, "C", problem_space,
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
set_argument(result, "m", problem_space, m);
set_argument(result, "n", problem_space, n);
set_argument(result, "k", problem_space, k);
set_argument(result, "split_k_slices", problem_space, split_k_slices);
set_argument(result, "batch_count", problem_space, batch_count);
set_argument(result, "alpha", problem_space,
library::lexical_cast(alpha, operation_desc.element_epilogue));
set_argument(result, "beta", problem_space,
library::lexical_cast(beta, operation_desc.element_epilogue));
}
/// Extracts the problem dimensions
Status SparseGemmOperationProfiler::initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SparseGemmDescription const &operation_desc =
static_cast<library::SparseGemmDescription const &>(operation->description());
if (operation_desc.gemm_kind != library::GemmKind::kSparse) {
return Status::kErrorInvalidProblem;
}
Status status = problem_.parse(operation_desc, problem_space, problem);
if (status != Status::kSuccess) {
return status;
}
gemm_workspace_.configuration.problem_size.m() = int(problem_.m);
gemm_workspace_.configuration.problem_size.n() = int(problem_.n);
gemm_workspace_.configuration.problem_size.k() = int(problem_.k);
gemm_workspace_.configuration.lda = problem_.lda;
gemm_workspace_.configuration.ldb = problem_.ldb;
gemm_workspace_.configuration.ldc = problem_.ldc;
gemm_workspace_.configuration.ldd = problem_.ldc;
gemm_workspace_.configuration.lde = problem_.lde;
gemm_workspace_.arguments.A = nullptr;
gemm_workspace_.arguments.B = nullptr;
gemm_workspace_.arguments.C = nullptr;
gemm_workspace_.arguments.D = nullptr;
gemm_workspace_.arguments.E = nullptr;
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
}
/// Initializes the performance result
void SparseGemmOperationProfiler::initialize_result_(
PerformanceResult &result,
Options const &options,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space) {
result.provider = library::Provider::kCUTLASS;
result.disposition = Disposition::kNotRun;
result.status = Status::kSuccess;
result.operation_name = operation_desc.name;
problem_.initialize_result(result, operation_desc, problem_space);
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
// Input bytes read and Output bytes written for the gemm problem
result.bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) *
problem_.k / problem_.sparse +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.n / 8) *
problem_.k +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) *
problem_.n +
int64_t(library::sizeof_bits(operation_desc.E.element) * problem_.m / 8) *
problem_.k / problem_.sparse / problem_.elements_per_128b;
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(problem_.beta.begin(), problem_.beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
result.bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
}
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
result.runtime = 0;
}
/// Initializes workspace
Status SparseGemmOperationProfiler::initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
library::SparseGemmDescription const &operation_desc =
static_cast<library::SparseGemmDescription const &>(operation->description());
if (options.execution_mode != ExecutionMode::kDryRun) {
gemm_workspace_.A = device_context.allocate_tensor(
options,
"A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k) / int(problem_.sparse)},
{int(problem_.lda)}
);
gemm_workspace_.B = device_context.allocate_tensor(
options,
"B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)}
);
gemm_workspace_.C = device_context.allocate_tensor(
options,
"C",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
);
gemm_workspace_.Computed = device_context.allocate_tensor(
"D",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
);
gemm_workspace_.E = device_context.allocate_sparsemeta_tensor(
options,
"E",
operation_desc.E.element,
operation_desc.E.layout,
operation_desc.A.element,
{int(problem_.m), int(problem_.k) / int(problem_.sparse) / int(problem_.elements_per_128b)},
{int(problem_.lde)}
);
gemm_workspace_.Reference = device_context.allocate_tensor(
"Reference",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)}
);
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
}
//
// Initialize the CUTLASS operation
//
Status status = Status::kSuccess;
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
if (options.execution_mode != ExecutionMode::kDryRun) {
uint64_t workspace_size = operation->get_host_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.host_workspace.resize(workspace_size, 0);
workspace_size = operation->get_device_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
status = operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
}
//
// If CUTLASS is enabled, generate a result for it
//
results_.push_back(model_result_);
results_.back().provider = library::Provider::kCUTLASS;
results_.back().op_kind = library::OperationKind::kSparseGemm;
results_.back().disposition = Disposition::kNotRun;
for(auto &verification_provider : options.verification.providers) {
results_.back().verification_map[verification_provider] = Disposition::kNotRun;
}
}
return status;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Verifies CUTLASS against references
bool SparseGemmOperationProfiler::verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
return true;
}
if (options.execution_mode == ExecutionMode::kDryRun) {
return true;
}
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.E = gemm_workspace_.E->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
//
// Run the CUTLASS operation
//
results_.back().status = operation->run(
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
if (results_.back().status != Status::kSuccess) {
results_.back().disposition = Disposition::kFailed;
return false;
}
cudaError_t result = cudaDeviceSynchronize();
if (result != cudaSuccess) {
results_.back().disposition = Disposition::kFailed;
return false;
}
// CUTLASS op ran the but not yet verified against any verification provider
results_.back().disposition = Disposition::kNotVerified;
//
// Run verification providers
//
if (options.verification.enabled) {
// Update disposition to worst case verification outcome among all
// verification providers which are supported
bool is_any_verification_run_passed = false;
for(auto &m : results_.back().verification_map) {
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
results_.back().disposition = m.second;
return true;
}
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
is_any_verification_run_passed = true;
}
}
if(is_any_verification_run_passed) {
results_.back().disposition = Disposition::kPassed;
}
}
// Return true means continue profiling
return true;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Measures performance results
bool SparseGemmOperationProfiler::profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem) {
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.E = gemm_workspace_.E->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
results_.back().status = profile_cutlass_(
results_.back().runtime,
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
return true;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -0,0 +1,208 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/* \file
\brief
*/
#pragma once
#include <vector>
#include <string>
#include <memory>
#include <algorithm>
#include <unordered_map>
// CUTLASS Library includes
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/library/manifest.h"
// Profiler includes
#include "options.h"
#include "device_context.h"
#include "operation_profiler.h"
#include "performance_result.h"
#include "problem_space.h"
#include "gemm_operation_profiler.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Abstract base class for each math function
class SparseGemmOperationProfiler : public OperationProfiler {
public:
/// Problem structure obtained from problem space
struct SparseGemmProblem {
int64_t m;
int64_t n;
int64_t k;
int64_t lda;
int64_t ldb;
int64_t ldc;
int64_t lde;
std::vector<uint8_t> alpha;
std::vector<uint8_t> beta;
int64_t split_k_slices;
int64_t batch_count;
static int const sparse = 2;
// every 128b ElementA uses one elementE
int elements_per_128b;
//
// Methods
//
SparseGemmProblem():
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), lde(0), split_k_slices(1), batch_count(1) { }
/// Parses the problem
Status parse(
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes a performance result
void initialize_result(
PerformanceResult &result,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/// Workspace used
struct SparseGemmWorkspace {
DeviceAllocation *A;
DeviceAllocation *B;
DeviceAllocation *C;
DeviceAllocation *E;
DeviceAllocation *Computed;
DeviceAllocation *Reference;
library::SparseGemmConfiguration configuration;
library::SparseGemmArguments arguments;
/// Buffer used for the operation's host workspace
std::vector<uint8_t> host_workspace;
/// Buffer used for the operations' device workspace
DeviceAllocation device_workspace;
//
// Methods
//
SparseGemmWorkspace():
A(nullptr), B(nullptr), C(nullptr), E(nullptr), Computed(nullptr), Reference(nullptr) { }
};
protected:
//
// Data members
//
// GEMM problem
SparseGemmProblem problem_;
/// Device memory allocations
SparseGemmWorkspace gemm_workspace_;
public:
//
// Methods
//
/// Ctor
SparseGemmOperationProfiler(Options const &options);
/// Destructor
virtual ~SparseGemmOperationProfiler();
/// Prints usage statement for the math function
virtual void print_usage(std::ostream &out) const;
/// Prints examples
virtual void print_examples(std::ostream &out) const;
/// Extracts the problem dimensions
virtual Status initialize_configuration(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Initializes workspace
virtual Status initialize_workspace(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Verifies CUTLASS against references
virtual bool verify_cutlass(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
/// Measures performance results
virtual bool profile(
Options const &options,
PerformanceReport &report,
DeviceContext &device_context,
library::Operation const *operation,
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
protected:
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
Options const &options,
library::SparseGemmDescription const &operation_desc,
ProblemSpace const &problem_space);
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace profiler
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////