CUTLASS 2.3 initial commit (#134)
CUTLASS 2.3 adds GEMMs targeting Sparse Tensor Cores on the NVIDIA Ampere Architecture, fast SGEMM, and small matrix classes, bug fixes, and performance enhancements.
This commit is contained in:
@@ -51,6 +51,18 @@ Status get_cutlass_status(cublasStatus_t cublas) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
|
||||
/// Converts a cuBLASS status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cublasStatus_t cublas_status) {
|
||||
|
||||
if (cublas_status == CUBLAS_STATUS_INVALID_VALUE) {
|
||||
return Disposition::kInvalidProblem;
|
||||
}
|
||||
else if (cublas_status == CUBLAS_STATUS_NOT_SUPPORTED) {
|
||||
return Disposition::kNotSupported;
|
||||
}
|
||||
return Disposition::kFailed;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
|
||||
bool get_cublas_transpose_operation(
|
||||
cublasOperation_t &operation,
|
||||
@@ -156,7 +168,6 @@ bool get_cublas_datatype(cublasDataType_t &data_type, library::NumericTypeID ele
|
||||
|
||||
/// Gets the cublas algorithm given threadblock tile dimensions and math opcode class
|
||||
cublasGemmAlgo_t get_cublas_gemm_algo(int cta_m, int cta_n, int cta_k, library::OpcodeClassID opcode_class) {
|
||||
// TODO
|
||||
return (opcode_class == library::OpcodeClassID::kSimt ?
|
||||
CUBLAS_GEMM_DEFAULT : CUBLAS_GEMM_DEFAULT_TENSOR_OP);
|
||||
}
|
||||
@@ -252,37 +263,69 @@ cublasGemmExDispatcher::cublasGemmExDispatcher(
|
||||
/// Executes GEMM using these arguments
|
||||
cublasStatus_t cublasGemmExDispatcher::operator()(cublasHandle_t handle) {
|
||||
|
||||
return cublasGemmEx(
|
||||
handle,
|
||||
trans_A,
|
||||
trans_B,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
arguments.alpha,
|
||||
arguments.A,
|
||||
data_type_A,
|
||||
int(configuration.lda),
|
||||
arguments.B,
|
||||
data_type_B,
|
||||
int(configuration.ldb),
|
||||
arguments.beta,
|
||||
arguments.D,
|
||||
data_type_C,
|
||||
int(configuration.ldc),
|
||||
#if (__CUDA_VER_MAJOR__ >= 11)
|
||||
compute_type,
|
||||
#else
|
||||
compute_data_type,
|
||||
#endif
|
||||
algo
|
||||
);
|
||||
if (configuration.mode == library::GemmUniversalMode::kBatched) {
|
||||
return cublasGemmStridedBatchedEx(
|
||||
handle,
|
||||
trans_A,
|
||||
trans_B,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
arguments.alpha,
|
||||
arguments.A,
|
||||
data_type_A,
|
||||
int(configuration.lda),
|
||||
arguments.batch_stride_A,
|
||||
arguments.B,
|
||||
data_type_B,
|
||||
int(configuration.ldb),
|
||||
arguments.batch_stride_B,
|
||||
arguments.beta,
|
||||
arguments.D,
|
||||
data_type_C,
|
||||
int(configuration.ldc),
|
||||
arguments.batch_stride_C,
|
||||
configuration.batch_count,
|
||||
#if (__CUDA_VER_MAJOR__ >= 11)
|
||||
compute_type,
|
||||
#else
|
||||
compute_data_type,
|
||||
#endif
|
||||
algo
|
||||
);
|
||||
}
|
||||
else {
|
||||
return cublasGemmEx(
|
||||
handle,
|
||||
trans_A,
|
||||
trans_B,
|
||||
configuration.problem_size.m(),
|
||||
configuration.problem_size.n(),
|
||||
configuration.problem_size.k(),
|
||||
arguments.alpha,
|
||||
arguments.A,
|
||||
data_type_A,
|
||||
int(configuration.lda),
|
||||
arguments.B,
|
||||
data_type_B,
|
||||
int(configuration.ldb),
|
||||
arguments.beta,
|
||||
arguments.D,
|
||||
data_type_C,
|
||||
int(configuration.ldc),
|
||||
#if (__CUDA_VER_MAJOR__ >= 11)
|
||||
compute_type,
|
||||
#else
|
||||
compute_data_type,
|
||||
#endif
|
||||
algo
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
@@ -47,6 +47,9 @@ namespace profiler {
|
||||
/// Converts a cuBLAS status to cutlass::Status
|
||||
Status get_cutlass_status(cublasStatus_t cublas);
|
||||
|
||||
/// Converts a cuBLASS status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cublasStatus_t cublas_status);
|
||||
|
||||
/// Maps a CUTLASS tensor layout to a cuBLAS transpose operation
|
||||
bool get_cublas_transpose_operation(
|
||||
cublasOperation_t &operation,
|
||||
|
||||
@@ -32,6 +32,8 @@
|
||||
// Profiler includes
|
||||
#include "cutlass_profiler.h"
|
||||
#include "gemm_operation_profiler.h"
|
||||
#include "sparse_gemm_operation_profiler.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
@@ -46,6 +48,8 @@ CutlassProfiler::CutlassProfiler(
|
||||
|
||||
operation_profilers_.emplace_back(new GemmOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new SparseGemmOperationProfiler(options));
|
||||
|
||||
}
|
||||
|
||||
CutlassProfiler::~CutlassProfiler() {
|
||||
|
||||
@@ -94,6 +94,12 @@ std::vector<int> DeviceAllocation::get_packed_layout(
|
||||
case library::LayoutTypeID::kRowMajor:
|
||||
stride = get_packed_layout_stride<cutlass::layout::RowMajor>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK2:
|
||||
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<2>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK2:
|
||||
stride = get_packed_layout_stride<cutlass::layout::RowMajorInterleaved<2>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK4:
|
||||
stride = get_packed_layout_stride<cutlass::layout::ColumnMajorInterleaved<4>>(extent);
|
||||
break;
|
||||
@@ -124,7 +130,9 @@ std::vector<int> DeviceAllocation::get_packed_layout(
|
||||
case library::LayoutTypeID::kTensorNHWC:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNHWC>(extent);
|
||||
break;
|
||||
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNDHWC>(extent);
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
@@ -200,6 +208,12 @@ size_t DeviceAllocation::construct_layout(
|
||||
case library::LayoutTypeID::kRowMajor:
|
||||
return construct_layout_<cutlass::layout::RowMajor>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK2:
|
||||
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<2>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK2:
|
||||
return construct_layout_<cutlass::layout::RowMajorInterleaved<2>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK4:
|
||||
return construct_layout_<cutlass::layout::ColumnMajorInterleaved<4>>(bytes, layout_id, extent, stride);
|
||||
|
||||
@@ -230,6 +244,9 @@ size_t DeviceAllocation::construct_layout(
|
||||
case library::LayoutTypeID::kTensorNHWC:
|
||||
return construct_layout_<cutlass::layout::TensorNHWC>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
return construct_layout_<cutlass::layout::TensorNDHWC>(bytes, layout_id, extent, stride);
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -240,9 +257,11 @@ size_t DeviceAllocation::construct_layout(
|
||||
|
||||
DeviceAllocation::DeviceAllocation():
|
||||
type_(library::NumericTypeID::kInvalid),
|
||||
batch_stride_(0),
|
||||
capacity_(0),
|
||||
pointer_(nullptr),
|
||||
layout_(library::LayoutTypeID::kUnknown) {
|
||||
layout_(library::LayoutTypeID::kUnknown),
|
||||
batch_count_(1) {
|
||||
|
||||
}
|
||||
|
||||
@@ -250,7 +269,8 @@ DeviceAllocation::DeviceAllocation(
|
||||
library::NumericTypeID type,
|
||||
size_t capacity
|
||||
):
|
||||
type_(type), capacity_(capacity), pointer_(nullptr), layout_(library::LayoutTypeID::kUnknown) {
|
||||
type_(type), batch_stride_(capacity), capacity_(capacity), pointer_(nullptr),
|
||||
layout_(library::LayoutTypeID::kUnknown), batch_count_(1) {
|
||||
|
||||
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity));
|
||||
|
||||
@@ -266,11 +286,12 @@ DeviceAllocation::DeviceAllocation(
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride
|
||||
std::vector<int> const &stride,
|
||||
int batch_count
|
||||
):
|
||||
type_(type), capacity_(size_t(0)), pointer_(nullptr) {
|
||||
type_(type), batch_stride_(size_t(0)), capacity_(size_t(0)), pointer_(nullptr), batch_count_(1) {
|
||||
|
||||
reset(type, layout_id, extent, stride);
|
||||
reset(type, layout_id, extent, stride, batch_count);
|
||||
}
|
||||
|
||||
DeviceAllocation::~DeviceAllocation() {
|
||||
@@ -285,12 +306,14 @@ DeviceAllocation &DeviceAllocation::reset() {
|
||||
}
|
||||
|
||||
type_ = library::NumericTypeID::kInvalid;
|
||||
batch_stride_ = 0;
|
||||
capacity_ = 0;
|
||||
pointer_ = nullptr;
|
||||
layout_ = library::LayoutTypeID::kUnknown;
|
||||
stride_.clear();
|
||||
extent_.clear();
|
||||
tensor_ref_buffer_.clear();
|
||||
batch_count_ = 1;
|
||||
|
||||
return *this;
|
||||
}
|
||||
@@ -299,16 +322,19 @@ DeviceAllocation &DeviceAllocation::reset(library::NumericTypeID type, size_t ca
|
||||
|
||||
reset();
|
||||
|
||||
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity));
|
||||
type_ = type;
|
||||
batch_stride_ = capacity;
|
||||
capacity_ = capacity;
|
||||
|
||||
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type_, capacity_));
|
||||
if (result != cudaSuccess) {
|
||||
throw std::bad_alloc();
|
||||
}
|
||||
|
||||
type_ = type;
|
||||
capacity_ = capacity;
|
||||
layout_ = library::LayoutTypeID::kUnknown;
|
||||
stride_.clear();
|
||||
extent_.clear();
|
||||
batch_count_ = 1;
|
||||
|
||||
tensor_ref_buffer_.resize(sizeof(pointer_), 0);
|
||||
std::memcpy(tensor_ref_buffer_.data(), &pointer_, sizeof(pointer_));
|
||||
@@ -321,7 +347,8 @@ DeviceAllocation &DeviceAllocation::reset(
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride) {
|
||||
std::vector<int> const &stride,
|
||||
int batch_count) {
|
||||
|
||||
reset();
|
||||
|
||||
@@ -332,13 +359,16 @@ DeviceAllocation &DeviceAllocation::reset(
|
||||
layout_ = layout_id;
|
||||
stride_ = stride;
|
||||
extent_ = extent;
|
||||
batch_count_ = batch_count;
|
||||
|
||||
capacity_ = construct_layout(
|
||||
batch_stride_ = construct_layout(
|
||||
tensor_ref_buffer_.data() + sizeof(pointer_),
|
||||
layout_id,
|
||||
extent,
|
||||
stride_);
|
||||
|
||||
capacity_ = batch_stride_ * batch_count_;
|
||||
|
||||
cudaError_t result = cudaMalloc((void **)&pointer_, bytes(type, capacity_));
|
||||
if (result != cudaSuccess) {
|
||||
throw std::bad_alloc();
|
||||
@@ -361,6 +391,10 @@ void *DeviceAllocation::data() const {
|
||||
return pointer_;
|
||||
}
|
||||
|
||||
void *DeviceAllocation::batch_data(int batch_idx) const {
|
||||
return static_cast<char *>(data()) + batch_stride_bytes() * batch_idx;
|
||||
}
|
||||
|
||||
library::LayoutTypeID DeviceAllocation::layout() const {
|
||||
return layout_;
|
||||
}
|
||||
@@ -374,6 +408,21 @@ std::vector<int> const & DeviceAllocation::extent() const {
|
||||
return extent_;
|
||||
}
|
||||
|
||||
/// Gets the number of adjacent tensors in memory
|
||||
int DeviceAllocation::batch_count() const {
|
||||
return batch_count_;
|
||||
}
|
||||
|
||||
/// Gets the stride (in units of elements) beteween items
|
||||
int64_t DeviceAllocation::batch_stride() const {
|
||||
return batch_stride_;
|
||||
}
|
||||
|
||||
/// Gets the stride (in units of bytes) beteween items
|
||||
int64_t DeviceAllocation::batch_stride_bytes() const {
|
||||
return bytes(type_, batch_stride_);
|
||||
}
|
||||
|
||||
size_t DeviceAllocation::capacity() const {
|
||||
return capacity_;
|
||||
}
|
||||
@@ -423,6 +472,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kBF16:
|
||||
cutlass::reference::device::BlockFillRandom<cutlass::bfloat16_t>(
|
||||
reinterpret_cast<cutlass::bfloat16_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kTF32:
|
||||
cutlass::reference::device::BlockFillRandom<cutlass::tfloat32_t>(
|
||||
reinterpret_cast<cutlass::tfloat32_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kF32:
|
||||
cutlass::reference::device::BlockFillRandom<float>(
|
||||
reinterpret_cast<float *>(pointer_),
|
||||
@@ -431,6 +496,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCBF16:
|
||||
cutlass::reference::device::BlockFillRandom<complex<bfloat16_t>>(
|
||||
reinterpret_cast<complex<bfloat16_t> *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCTF32:
|
||||
cutlass::reference::device::BlockFillRandom<cutlass::complex<cutlass::tfloat32_t>>(
|
||||
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCF32:
|
||||
cutlass::reference::device::BlockFillRandom<cutlass::complex<float>>(
|
||||
reinterpret_cast<cutlass::complex<float> *>(pointer_),
|
||||
@@ -455,6 +536,22 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS2:
|
||||
cutlass::reference::device::BlockFillRandom<int2b_t>(
|
||||
reinterpret_cast<int2b_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS4:
|
||||
cutlass::reference::device::BlockFillRandom<int4b_t>(
|
||||
reinterpret_cast<int4b_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS8:
|
||||
cutlass::reference::device::BlockFillRandom<int8_t>(
|
||||
reinterpret_cast<int8_t *>(pointer_),
|
||||
@@ -487,6 +584,30 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kB1:
|
||||
cutlass::reference::device::BlockFillRandom<uint1b_t>(
|
||||
reinterpret_cast<uint1b_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU2:
|
||||
cutlass::reference::device::BlockFillRandom<uint2b_t>(
|
||||
reinterpret_cast<uint2b_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU4:
|
||||
cutlass::reference::device::BlockFillRandom<uint4b_t>(
|
||||
reinterpret_cast<uint4b_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU8:
|
||||
cutlass::reference::device::BlockFillRandom<uint8_t>(
|
||||
reinterpret_cast<uint8_t *>(pointer_),
|
||||
@@ -523,7 +644,6 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
if (!good()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
@@ -540,6 +660,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kBF16:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::bfloat16_t>(
|
||||
reinterpret_cast<cutlass::bfloat16_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kTF32:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::tfloat32_t>(
|
||||
reinterpret_cast<cutlass::tfloat32_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kF32:
|
||||
cutlass::reference::host::BlockFillRandom<float>(
|
||||
reinterpret_cast<float *>(host_data.data()),
|
||||
@@ -556,6 +692,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCBF16:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::complex<cutlass::bfloat16_t>>(
|
||||
reinterpret_cast<cutlass::complex<cutlass::bfloat16_t> *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCTF32:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::complex<cutlass::tfloat32_t>>(
|
||||
reinterpret_cast<cutlass::complex<cutlass::tfloat32_t> *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kCF32:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::complex<float>>(
|
||||
reinterpret_cast<cutlass::complex<float> *>(host_data.data()),
|
||||
@@ -580,6 +732,22 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS2:
|
||||
cutlass::reference::host::BlockFillRandom<int2b_t>(
|
||||
reinterpret_cast<int2b_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS4:
|
||||
cutlass::reference::host::BlockFillRandom<int4b_t>(
|
||||
reinterpret_cast<int4b_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS8:
|
||||
cutlass::reference::host::BlockFillRandom<int8_t>(
|
||||
reinterpret_cast<int8_t *>(host_data.data()),
|
||||
@@ -612,6 +780,30 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kB1:
|
||||
cutlass::reference::host::BlockFillRandom<uint1b_t>(
|
||||
reinterpret_cast<uint1b_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU2:
|
||||
cutlass::reference::host::BlockFillRandom<uint2b_t>(
|
||||
reinterpret_cast<uint2b_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU4:
|
||||
cutlass::reference::host::BlockFillRandom<uint4b_t>(
|
||||
reinterpret_cast<uint4b_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU8:
|
||||
cutlass::reference::host::BlockFillRandom<uint8_t>(
|
||||
reinterpret_cast<uint8_t *>(host_data.data()),
|
||||
@@ -650,6 +842,67 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
copy_from_host(host_data.data());
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_sparsemeta_device(int seed, int MetaSizeInBits) {
|
||||
if (!good()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
// Instantiate calls to CURAND here. This file takes a long time to compile for
|
||||
// this reason.
|
||||
|
||||
switch (type_) {
|
||||
case library::NumericTypeID::kU16:
|
||||
cutlass::reference::device::BlockFillRandomSparseMeta<uint16_t>(
|
||||
reinterpret_cast<uint16_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
MetaSizeInBits
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kU32:
|
||||
cutlass::reference::device::BlockFillRandomSparseMeta<uint32_t>(
|
||||
reinterpret_cast<uint32_t *>(pointer_),
|
||||
capacity_,
|
||||
seed,
|
||||
MetaSizeInBits
|
||||
);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_sparsemeta_host(int seed, int MetaSizeInBits) {
|
||||
if (!good()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
std::vector<uint8_t> host_data(bytes());
|
||||
|
||||
switch (type_) {
|
||||
case library::NumericTypeID::kS16:
|
||||
cutlass::reference::host::BlockFillRandomSparseMeta<uint16_t>(
|
||||
reinterpret_cast<uint16_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
MetaSizeInBits
|
||||
);
|
||||
break;
|
||||
case library::NumericTypeID::kS32:
|
||||
cutlass::reference::host::BlockFillRandomSparseMeta<uint32_t>(
|
||||
reinterpret_cast<uint32_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
MetaSizeInBits
|
||||
);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
copy_from_host(host_data.data());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns true if two blocks have exactly the same value
|
||||
@@ -666,6 +919,18 @@ bool DeviceAllocation::block_compare_equal(
|
||||
reinterpret_cast<half_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kBF16:
|
||||
return reference::device::BlockCompareEqual<bfloat16_t>(
|
||||
reinterpret_cast<bfloat16_t const *>(ptr_A),
|
||||
reinterpret_cast<bfloat16_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kTF32:
|
||||
return reference::device::BlockCompareEqual<tfloat32_t>(
|
||||
reinterpret_cast<tfloat32_t const *>(ptr_A),
|
||||
reinterpret_cast<tfloat32_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kF32:
|
||||
return reference::device::BlockCompareEqual<float>(
|
||||
reinterpret_cast<float const *>(ptr_A),
|
||||
@@ -684,6 +949,18 @@ bool DeviceAllocation::block_compare_equal(
|
||||
reinterpret_cast<complex<half_t> const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kCBF16:
|
||||
return reference::device::BlockCompareEqual<complex<bfloat16_t>>(
|
||||
reinterpret_cast<complex<bfloat16_t> const *>(ptr_A),
|
||||
reinterpret_cast<complex<bfloat16_t> const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kCTF32:
|
||||
return reference::device::BlockCompareEqual<complex<tfloat32_t>>(
|
||||
reinterpret_cast<complex<tfloat32_t> const *>(ptr_A),
|
||||
reinterpret_cast<complex<tfloat32_t> const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kF64:
|
||||
return reference::device::BlockCompareEqual<double>(
|
||||
reinterpret_cast<double const *>(ptr_A),
|
||||
@@ -695,6 +972,18 @@ bool DeviceAllocation::block_compare_equal(
|
||||
reinterpret_cast<complex<double> const *>(ptr_A),
|
||||
reinterpret_cast<complex<double> const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kS2:
|
||||
return reference::device::BlockCompareEqual<int2b_t>(
|
||||
reinterpret_cast<int2b_t const *>(ptr_A),
|
||||
reinterpret_cast<int2b_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kS4:
|
||||
return reference::device::BlockCompareEqual<int4b_t>(
|
||||
reinterpret_cast<int4b_t const *>(ptr_A),
|
||||
reinterpret_cast<int4b_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kS8:
|
||||
return reference::device::BlockCompareEqual<int8_t>(
|
||||
@@ -719,6 +1008,24 @@ bool DeviceAllocation::block_compare_equal(
|
||||
reinterpret_cast<int64_t const *>(ptr_A),
|
||||
reinterpret_cast<int64_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kB1:
|
||||
return reference::device::BlockCompareEqual<uint1b_t>(
|
||||
reinterpret_cast<uint1b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint1b_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kU2:
|
||||
return reference::device::BlockCompareEqual<uint2b_t>(
|
||||
reinterpret_cast<uint2b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint2b_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kU4:
|
||||
return reference::device::BlockCompareEqual<uint4b_t>(
|
||||
reinterpret_cast<uint4b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint4b_t const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kU8:
|
||||
return reference::device::BlockCompareEqual<uint8_t>(
|
||||
@@ -767,6 +1074,22 @@ bool DeviceAllocation::block_compare_relatively_equal(
|
||||
static_cast<half_t>(epsilon),
|
||||
static_cast<half_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kBF16:
|
||||
return reference::device::BlockCompareRelativelyEqual<bfloat16_t>(
|
||||
reinterpret_cast<bfloat16_t const *>(ptr_A),
|
||||
reinterpret_cast<bfloat16_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<bfloat16_t>(epsilon),
|
||||
static_cast<bfloat16_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kTF32:
|
||||
return reference::device::BlockCompareRelativelyEqual<tfloat32_t>(
|
||||
reinterpret_cast<tfloat32_t const *>(ptr_A),
|
||||
reinterpret_cast<tfloat32_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<tfloat32_t>(epsilon),
|
||||
static_cast<tfloat32_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kF32:
|
||||
return reference::device::BlockCompareRelativelyEqual<float>(
|
||||
reinterpret_cast<float const *>(ptr_A),
|
||||
@@ -782,6 +1105,22 @@ bool DeviceAllocation::block_compare_relatively_equal(
|
||||
capacity,
|
||||
static_cast<double>(epsilon),
|
||||
static_cast<double>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kS2:
|
||||
return reference::device::BlockCompareRelativelyEqual<int2b_t>(
|
||||
reinterpret_cast<int2b_t const *>(ptr_A),
|
||||
reinterpret_cast<int2b_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<int2b_t>(epsilon),
|
||||
static_cast<int2b_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kS4:
|
||||
return reference::device::BlockCompareRelativelyEqual<int4b_t>(
|
||||
reinterpret_cast<int4b_t const *>(ptr_A),
|
||||
reinterpret_cast<int4b_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<int4b_t>(epsilon),
|
||||
static_cast<int4b_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kS8:
|
||||
return reference::device::BlockCompareRelativelyEqual<int8_t>(
|
||||
@@ -814,6 +1153,30 @@ bool DeviceAllocation::block_compare_relatively_equal(
|
||||
capacity,
|
||||
static_cast<int64_t>(epsilon),
|
||||
static_cast<int64_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kB1:
|
||||
return reference::device::BlockCompareRelativelyEqual<uint1b_t>(
|
||||
reinterpret_cast<uint1b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint1b_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<uint1b_t>(epsilon),
|
||||
static_cast<uint1b_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kU2:
|
||||
return reference::device::BlockCompareRelativelyEqual<uint2b_t>(
|
||||
reinterpret_cast<uint2b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint2b_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<uint2b_t>(epsilon),
|
||||
static_cast<uint2b_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kU4:
|
||||
return reference::device::BlockCompareRelativelyEqual<uint4b_t>(
|
||||
reinterpret_cast<uint4b_t const *>(ptr_A),
|
||||
reinterpret_cast<uint4b_t const *>(ptr_B),
|
||||
capacity,
|
||||
static_cast<uint4b_t>(epsilon),
|
||||
static_cast<uint4b_t>(nonzero_floor));
|
||||
|
||||
case library::NumericTypeID::kU8:
|
||||
return reference::device::BlockCompareRelativelyEqual<uint8_t>(
|
||||
@@ -852,6 +1215,12 @@ bool DeviceAllocation::block_compare_relatively_equal(
|
||||
// As a simplification, we can require bitwise equality. This avoids false positives.
|
||||
// (i.e. "pass" really means passing. "Fail" may not actually mean failure given appropriate epsilon.)
|
||||
//
|
||||
case library::NumericTypeID::kCF16:
|
||||
return reference::device::BlockCompareEqual<cutlass::complex<half_t> >(
|
||||
reinterpret_cast<complex<half_t> const *>(ptr_A),
|
||||
reinterpret_cast<complex<half_t> const *>(ptr_B),
|
||||
capacity);
|
||||
|
||||
case library::NumericTypeID::kCF32:
|
||||
return reference::device::BlockCompareEqual<cutlass::complex<float> >(
|
||||
reinterpret_cast<complex<float> const *>(ptr_A),
|
||||
@@ -865,7 +1234,9 @@ bool DeviceAllocation::block_compare_relatively_equal(
|
||||
capacity);
|
||||
|
||||
default:
|
||||
throw std::runtime_error("Unsupported numeric type");
|
||||
{
|
||||
throw std::runtime_error(std::string("Unsupported numeric type: ") + to_string(numeric_type));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -928,13 +1299,17 @@ static void write_tensor_csv_static_tensor_view(
|
||||
Layout layout(stride);
|
||||
HostTensor<Element, Layout> host_tensor(extent, layout, false);
|
||||
|
||||
if (host_tensor.capacity() != allocation.capacity()) {
|
||||
if (host_tensor.capacity() != allocation.batch_stride()) {
|
||||
throw std::runtime_error("Unexpected capacity to equal.");
|
||||
}
|
||||
|
||||
host_tensor.copy_in_device_to_host(static_cast<Element const *>(allocation.data()), host_tensor.capacity());
|
||||
host_tensor.copy_in_device_to_host(
|
||||
static_cast<Element const *>(allocation.data()),
|
||||
allocation.batch_stride());
|
||||
|
||||
TensorViewWrite(out, host_tensor.host_view());
|
||||
|
||||
out << "\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -951,9 +1326,42 @@ static void write_tensor_csv_static_type(
|
||||
case library::LayoutTypeID::kColumnMajor:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajor>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK2:
|
||||
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<2>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK2:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<2>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK4:
|
||||
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<4>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK4:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<4>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK16:
|
||||
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<16>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK16:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<16>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kRowMajorInterleavedK64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::RowMajorInterleaved<64>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kColumnMajorInterleavedK64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::ColumnMajorInterleaved<64>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNHWC:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNHWC>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNDHWC>(out, allocation);
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error("Unhandled layout");
|
||||
}
|
||||
@@ -970,6 +1378,14 @@ void DeviceAllocation::write_tensor_csv(
|
||||
write_tensor_csv_static_type<half_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kBF16:
|
||||
write_tensor_csv_static_type<bfloat16_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kTF32:
|
||||
write_tensor_csv_static_type<tfloat32_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kF32:
|
||||
write_tensor_csv_static_type<float>(out, *this);
|
||||
break;
|
||||
@@ -977,6 +1393,14 @@ void DeviceAllocation::write_tensor_csv(
|
||||
case library::NumericTypeID::kF64:
|
||||
write_tensor_csv_static_type<double>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS2:
|
||||
write_tensor_csv_static_type<int2b_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS4:
|
||||
write_tensor_csv_static_type<int4b_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS8:
|
||||
write_tensor_csv_static_type<int8_t>(out, *this);
|
||||
@@ -993,6 +1417,18 @@ void DeviceAllocation::write_tensor_csv(
|
||||
case library::NumericTypeID::kS64:
|
||||
write_tensor_csv_static_type<int64_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kB1:
|
||||
write_tensor_csv_static_type<uint1b_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU2:
|
||||
write_tensor_csv_static_type<uint2b_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU4:
|
||||
write_tensor_csv_static_type<uint4b_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU8:
|
||||
write_tensor_csv_static_type<uint8_t>(out, *this);
|
||||
@@ -1010,6 +1446,10 @@ void DeviceAllocation::write_tensor_csv(
|
||||
write_tensor_csv_static_type<uint64_t>(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kCF16:
|
||||
write_tensor_csv_static_type<cutlass::complex<half_t> >(out, *this);
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kCF32:
|
||||
write_tensor_csv_static_type<cutlass::complex<float> >(out, *this);
|
||||
break;
|
||||
|
||||
@@ -51,6 +51,9 @@ private:
|
||||
/// Data type of contained elements
|
||||
library::NumericTypeID type_;
|
||||
|
||||
/// Gets the stride between elements
|
||||
size_t batch_stride_;
|
||||
|
||||
/// Capacity in elements of device allocation
|
||||
size_t capacity_;
|
||||
|
||||
@@ -66,6 +69,9 @@ private:
|
||||
/// Extent vector
|
||||
std::vector<int> extent_;
|
||||
|
||||
/// Support allocating a 'batch' of non-overlapping tensors in contiguous memory
|
||||
int batch_count_;
|
||||
|
||||
/// Buffer holding TensorRef instance to recently allocated memory
|
||||
std::vector<uint8_t> tensor_ref_buffer_;
|
||||
|
||||
@@ -118,7 +124,8 @@ public:
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride = std::vector<int>());
|
||||
std::vector<int> const &stride = std::vector<int>(),
|
||||
int batch_count = 1);
|
||||
|
||||
~DeviceAllocation();
|
||||
|
||||
@@ -132,7 +139,8 @@ public:
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride = std::vector<int>());
|
||||
std::vector<int> const &stride = std::vector<int>(),
|
||||
int batch_count = 1);
|
||||
|
||||
/// Returns a buffer owning the tensor reference
|
||||
std::vector<uint8_t> &tensor_ref() {
|
||||
@@ -144,9 +152,12 @@ public:
|
||||
/// Data type of contained elements
|
||||
library::NumericTypeID type() const;
|
||||
|
||||
/// Pointer to device memory
|
||||
/// Pointer to start of device memory allocation
|
||||
void *data() const;
|
||||
|
||||
/// Pointer to the first element of a batch
|
||||
void *batch_data(int batch_idx) const;
|
||||
|
||||
/// Gets the layout type
|
||||
library::LayoutTypeID layout() const;
|
||||
|
||||
@@ -156,6 +167,15 @@ public:
|
||||
/// Gets the extent vector
|
||||
std::vector<int> const & extent() const;
|
||||
|
||||
/// Gets the number of adjacent tensors in memory
|
||||
int batch_count() const;
|
||||
|
||||
/// Gets the stride (in units of elements) beteween items
|
||||
int64_t batch_stride() const;
|
||||
|
||||
/// Gets the stride (in units of bytes) beteween items
|
||||
int64_t batch_stride_bytes() const;
|
||||
|
||||
/// Capacity of allocation in number of elements
|
||||
size_t capacity() const;
|
||||
|
||||
@@ -165,9 +185,15 @@ public:
|
||||
/// Initializes a device allocation to a random distribution using cuRAND
|
||||
void initialize_random_device(int seed, Distribution dist);
|
||||
|
||||
/// Initializes a device allocation to a random distribution using cuRAND
|
||||
/// Initializes a host allocation to a random distribution using std::cout
|
||||
void initialize_random_host(int seed, Distribution dist);
|
||||
|
||||
/// Initializes a device allocation to a random distribution using cuRAND
|
||||
void initialize_random_sparsemeta_device(int seed, int MetaSizeInBits);
|
||||
|
||||
/// Initializes a host allocation to a random distribution using std::cout
|
||||
void initialize_random_sparsemeta_host(int seed, int MetaSizeInBits);
|
||||
|
||||
/// Copies from an equivalent-sized tensor in device memory
|
||||
void copy_from_device(void const *ptr);
|
||||
|
||||
|
||||
@@ -52,9 +52,10 @@ DeviceAllocation *DeviceContext::allocate_tensor(
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride) {
|
||||
std::vector<int> const &stride,
|
||||
int batch_count) {
|
||||
|
||||
device_memory_.emplace_back(type, layout_id, extent, stride);
|
||||
device_memory_.emplace_back(type, layout_id, extent, stride, batch_count);
|
||||
DeviceAllocation *allocation = &device_memory_.back();
|
||||
|
||||
allocations_[name] = allocation;
|
||||
@@ -68,10 +69,11 @@ DeviceAllocation *DeviceContext::allocate_tensor(
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride) {
|
||||
std::vector<int> const &stride,
|
||||
int batch_count) {
|
||||
|
||||
DeviceAllocation *allocation =
|
||||
allocate_tensor(name, type, layout_id, extent, stride);
|
||||
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
|
||||
|
||||
if (options.initialization.enabled) {
|
||||
Distribution data_distribution = options.initialization.data_distribution;
|
||||
@@ -81,11 +83,23 @@ DeviceAllocation *DeviceContext::allocate_tensor(
|
||||
// change data distribution based on bit width
|
||||
switch(type) {
|
||||
case library::NumericTypeID::kB1:
|
||||
data_distribution.set_uniform(0, 2, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kS8:
|
||||
data_distribution.set_uniform(0, 1, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kS2:
|
||||
data_distribution.set_uniform(-1, 1, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kS4:
|
||||
data_distribution.set_uniform(-2, 2, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kU2:
|
||||
data_distribution.set_uniform(0, 2, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kU4:
|
||||
data_distribution.set_uniform(0, 2, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kS8:
|
||||
data_distribution.set_uniform(-3, 3, 0);
|
||||
break;
|
||||
case library::NumericTypeID::kU8:
|
||||
data_distribution.set_uniform(0, 4, 0);
|
||||
break;
|
||||
@@ -96,18 +110,50 @@ DeviceAllocation *DeviceContext::allocate_tensor(
|
||||
if (options.initialization.provider == library::Provider::kReferenceDevice) {
|
||||
allocation->initialize_random_device(
|
||||
options.initialization.seed,
|
||||
data_distribution);
|
||||
data_distribution);
|
||||
}
|
||||
else if (options.initialization.provider == library::Provider::kReferenceHost) {
|
||||
allocation->initialize_random_host(
|
||||
options.initialization.seed,
|
||||
data_distribution);
|
||||
data_distribution);
|
||||
}
|
||||
}
|
||||
|
||||
return allocation;
|
||||
}
|
||||
|
||||
/// Allocates memory for sparse meta data
|
||||
DeviceAllocation *DeviceContext::allocate_sparsemeta_tensor(
|
||||
Options const &options,
|
||||
std::string const &name,
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
library::NumericTypeID type_a,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride,
|
||||
int batch_count) {
|
||||
|
||||
DeviceAllocation *allocation =
|
||||
allocate_tensor(name, type, layout_id, extent, stride, batch_count);
|
||||
|
||||
if (options.initialization.enabled) {
|
||||
// TF32 has 4bit meta data. The rest has 2bit.
|
||||
int MetaSizeInBits = (cutlass::library::sizeof_bits(type_a) == 32) ? 4 : 2;
|
||||
|
||||
if (options.initialization.provider == library::Provider::kReferenceDevice) {
|
||||
allocation->initialize_random_sparsemeta_device(
|
||||
options.initialization.seed,
|
||||
MetaSizeInBits);
|
||||
}
|
||||
else if (options.initialization.provider == library::Provider::kReferenceHost) {
|
||||
allocation->initialize_random_sparsemeta_host(
|
||||
options.initialization.seed,
|
||||
MetaSizeInBits);
|
||||
}
|
||||
}
|
||||
|
||||
return allocation;
|
||||
}
|
||||
/// Clears named allocations (but does not necessarily free memory)
|
||||
void DeviceContext::clear() {
|
||||
allocations_.clear();
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
|
||||
#include "options.h"
|
||||
#include "device_allocation.h"
|
||||
@@ -76,7 +77,8 @@ public:
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride = std::vector<int>());
|
||||
std::vector<int> const &stride = std::vector<int>(),
|
||||
int batch_count = 1);
|
||||
|
||||
/// Allocates memory of a given type, capacity (elements), and name
|
||||
DeviceAllocation *allocate_tensor(
|
||||
@@ -85,7 +87,19 @@ public:
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride = std::vector<int>());
|
||||
std::vector<int> const &stride = std::vector<int>(),
|
||||
int batch_count = 1);
|
||||
|
||||
/// Allocates memory for sparse meta data
|
||||
DeviceAllocation *allocate_sparsemeta_tensor(
|
||||
Options const &options,
|
||||
std::string const &name,
|
||||
library::NumericTypeID type,
|
||||
library::LayoutTypeID layout_id,
|
||||
library::NumericTypeID type_a,
|
||||
std::vector<int> const &extent,
|
||||
std::vector<int> const &stride = std::vector<int>(),
|
||||
int batch_count = 1);
|
||||
|
||||
/// Clears named allocations (but does not necessarily free memory)
|
||||
void clear();
|
||||
|
||||
@@ -37,6 +37,9 @@
|
||||
#include "gemm_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/handle.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
@@ -135,6 +138,8 @@ Status GemmOperationProfiler::GemmProblem::parse(
|
||||
library::GemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
this->mode = library::GemmUniversalMode::kGemm;
|
||||
|
||||
if (!arg_as_int(this->m, "m", problem_space, problem)) {
|
||||
// default value
|
||||
@@ -151,6 +156,7 @@ Status GemmOperationProfiler::GemmProblem::parse(
|
||||
this->k = 1024;
|
||||
}
|
||||
|
||||
this->mode = library::GemmUniversalMode::kGemm;
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
@@ -160,6 +166,14 @@ Status GemmOperationProfiler::GemmProblem::parse(
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
else if (this->batch_count > 1) {
|
||||
this->mode = library::GemmUniversalMode::kBatched;
|
||||
}
|
||||
|
||||
if (this->split_k_slices > 1 && this->batch_count > 1) {
|
||||
// At least one of these must be one
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
@@ -209,6 +223,48 @@ Status GemmOperationProfiler::GemmProblem::parse(
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
int64_t bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
|
||||
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
|
||||
}
|
||||
|
||||
bytes *= batch_count;
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
|
||||
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
flops_ *= 4;
|
||||
break;
|
||||
|
||||
case library::MathOperationID::kMultiplyAddGaussianComplex:
|
||||
flops_ *= 3;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
return flops_;
|
||||
}
|
||||
|
||||
|
||||
/// Initializes a performance result
|
||||
void GemmOperationProfiler::GemmProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
@@ -266,6 +322,7 @@ Status GemmOperationProfiler::initialize_configuration(
|
||||
return status;
|
||||
}
|
||||
|
||||
gemm_workspace_.configuration.mode = problem_.mode;
|
||||
gemm_workspace_.configuration.problem_size.m() = int(problem_.m);
|
||||
gemm_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
gemm_workspace_.configuration.problem_size.k() = int(problem_.k);
|
||||
@@ -273,8 +330,13 @@ Status GemmOperationProfiler::initialize_configuration(
|
||||
gemm_workspace_.configuration.ldb = problem_.ldb;
|
||||
gemm_workspace_.configuration.ldc = problem_.ldc;
|
||||
gemm_workspace_.configuration.ldd = problem_.ldc;
|
||||
//gemm_workspace_.configuration.split_k_slices = int(problem_.split_k_slices);
|
||||
gemm_workspace_.configuration.batch_count = int(problem_.split_k_slices);
|
||||
|
||||
if (problem_.mode == library::GemmUniversalMode::kBatched) {
|
||||
gemm_workspace_.configuration.batch_count = problem_.batch_count;
|
||||
}
|
||||
else {
|
||||
gemm_workspace_.configuration.batch_count = problem_.split_k_slices;
|
||||
}
|
||||
|
||||
gemm_workspace_.arguments.A = nullptr;
|
||||
gemm_workspace_.arguments.B = nullptr;
|
||||
@@ -305,32 +367,10 @@ void GemmOperationProfiler::initialize_result_(
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
result.bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) * problem_.k +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.n / 8) * problem_.k +
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
|
||||
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(problem_.beta.begin(), problem_.beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
result.bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
|
||||
}
|
||||
|
||||
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
|
||||
result.bytes = problem_.bytes(operation_desc);
|
||||
result.flops = problem_.flops(operation_desc);
|
||||
result.runtime = 0;
|
||||
|
||||
// complex-valued support
|
||||
switch (operation_desc.tile_description.math_instruction.math_operation) {
|
||||
case library::MathOperationID::kMultiplyAddComplex:
|
||||
result.flops *= 4;
|
||||
break;
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
@@ -345,6 +385,21 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
library::GemmDescription const &operation_desc =
|
||||
static_cast<library::GemmDescription const &>(operation->description());
|
||||
|
||||
// Compute the number of copies of the problem to avoid L2 camping.
|
||||
if (!options.profiling.workspace_count) {
|
||||
int64_t bytes = problem_.bytes(operation_desc);
|
||||
if (bytes < 3 * int64_t(options.device.properties.l2CacheSize)) {
|
||||
gemm_workspace_.problem_count =
|
||||
1 + int((3 * int64_t(options.device.properties.l2CacheSize)) / bytes);
|
||||
}
|
||||
else {
|
||||
gemm_workspace_.problem_count = 1;
|
||||
}
|
||||
}
|
||||
else {
|
||||
gemm_workspace_.problem_count = options.profiling.workspace_count;
|
||||
}
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
gemm_workspace_.A = device_context.allocate_tensor(
|
||||
@@ -353,7 +408,8 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.m), int(problem_.k)},
|
||||
{int(problem_.lda)}
|
||||
{int(problem_.lda)},
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.B = device_context.allocate_tensor(
|
||||
@@ -362,7 +418,8 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
operation_desc.B.element,
|
||||
operation_desc.B.layout,
|
||||
{int(problem_.k), int(problem_.n)},
|
||||
{int(problem_.ldb)}
|
||||
{int(problem_.ldb)},
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.C = device_context.allocate_tensor(
|
||||
@@ -371,7 +428,8 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
{int(problem_.ldc)},
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.Computed = device_context.allocate_tensor(
|
||||
@@ -379,7 +437,8 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
{int(problem_.ldc)},
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.Reference = device_context.allocate_tensor(
|
||||
@@ -387,7 +446,8 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
{int(problem_.ldc)},
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
|
||||
@@ -458,6 +518,10 @@ bool GemmOperationProfiler::verify_cutlass(
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
@@ -512,6 +576,8 @@ bool GemmOperationProfiler::verify_cutlass(
|
||||
}
|
||||
}
|
||||
#endif // #if CUTLASS_ENABLE_CUBLAS
|
||||
|
||||
verify_with_reference_(options, report, device_context, operation, problem_space, problem);
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
@@ -561,7 +627,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
|
||||
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -589,9 +655,13 @@ bool GemmOperationProfiler::verify_with_cublas_(
|
||||
|
||||
// Initialize structure containing GEMM arguments
|
||||
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
|
||||
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
|
||||
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
|
||||
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
|
||||
gemm_workspace_.arguments.C = gemm_workspace_.Reference->data();
|
||||
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Reference->batch_stride();
|
||||
gemm_workspace_.arguments.D = gemm_workspace_.Reference->data();
|
||||
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Reference->batch_stride();
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
@@ -615,7 +685,7 @@ bool GemmOperationProfiler::verify_with_cublas_(
|
||||
// Handle errors
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = Disposition::kFailed;
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = get_cutlass_disposition(status);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -626,7 +696,8 @@ bool GemmOperationProfiler::verify_with_cublas_(
|
||||
results_.back().verification_map[library::Provider::kCUBLAS] = compare_tensors(
|
||||
options,
|
||||
*gemm_workspace_.Computed,
|
||||
*gemm_workspace_.Reference
|
||||
*gemm_workspace_.Reference,
|
||||
gemm_workspace_.Computed->batch_stride()
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
@@ -653,6 +724,150 @@ bool GemmOperationProfiler::verify_with_cublas_(
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against host and device references
|
||||
bool GemmOperationProfiler::verify_with_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::GemmDescription const &gemm_desc =
|
||||
static_cast<library::GemmDescription const &>(operation->description());
|
||||
|
||||
//
|
||||
// Initialize state
|
||||
//
|
||||
|
||||
library::Provider references[] = {
|
||||
library::Provider::kReferenceDevice,
|
||||
library::Provider::kReferenceHost
|
||||
};
|
||||
|
||||
for (auto provider : references) {
|
||||
|
||||
// Skip providers that are not enabled
|
||||
if (!options.verification.provider_enabled(provider)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
void *ptr_A = gemm_workspace_.A->data();
|
||||
void *ptr_B = gemm_workspace_.B->data();
|
||||
void *ptr_C = gemm_workspace_.C->data();
|
||||
void *ptr_D = gemm_workspace_.Reference->data();
|
||||
|
||||
// To support the host-side reference, conditionally allocate and
|
||||
// copy tensors to host memory.
|
||||
std::vector<uint8_t> host_data_A;
|
||||
std::vector<uint8_t> host_data_B;
|
||||
std::vector<uint8_t> host_data_C;
|
||||
std::vector<uint8_t> host_data_D;
|
||||
|
||||
if (provider == library::Provider::kReferenceHost) {
|
||||
|
||||
host_data_A.resize(gemm_workspace_.A->bytes());
|
||||
ptr_A = host_data_A.data();
|
||||
gemm_workspace_.A->copy_to_host(ptr_A);
|
||||
|
||||
host_data_B.resize(gemm_workspace_.B->bytes());
|
||||
ptr_B = host_data_B.data();
|
||||
gemm_workspace_.B->copy_to_host(ptr_B);
|
||||
|
||||
host_data_C.resize(gemm_workspace_.C->bytes());
|
||||
ptr_C = host_data_C.data();
|
||||
gemm_workspace_.C->copy_to_host(ptr_C);
|
||||
|
||||
host_data_D.resize(gemm_workspace_.Reference->bytes());
|
||||
ptr_D = host_data_D.data();
|
||||
}
|
||||
|
||||
//
|
||||
// Launch
|
||||
//
|
||||
|
||||
library::Handle handle;
|
||||
|
||||
handle.set_provider(provider);
|
||||
|
||||
Status status = handle.gemm_universal(
|
||||
library::GemmUniversalMode::kGemm,
|
||||
gemm_workspace_.configuration.problem_size.m(),
|
||||
gemm_workspace_.configuration.problem_size.n(),
|
||||
gemm_workspace_.configuration.problem_size.k(),
|
||||
gemm_desc.tile_description.math_instruction.element_accumulator,
|
||||
gemm_desc.element_epilogue,
|
||||
|
||||
problem_.alpha.data(),
|
||||
|
||||
gemm_desc.A.element,
|
||||
gemm_desc.A.layout,
|
||||
gemm_desc.transform_A,
|
||||
ptr_A,
|
||||
int(gemm_workspace_.configuration.lda),
|
||||
|
||||
gemm_desc.B.element,
|
||||
gemm_desc.B.layout,
|
||||
gemm_desc.transform_B,
|
||||
ptr_B,
|
||||
int(gemm_workspace_.configuration.ldb),
|
||||
|
||||
problem_.beta.data(),
|
||||
|
||||
gemm_desc.C.element,
|
||||
ptr_C,
|
||||
int(gemm_workspace_.configuration.ldc),
|
||||
|
||||
ptr_D,
|
||||
int(gemm_workspace_.configuration.ldd),
|
||||
|
||||
gemm_workspace_.configuration.batch_count,
|
||||
gemm_workspace_.A->batch_stride(),
|
||||
gemm_workspace_.B->batch_stride(),
|
||||
gemm_workspace_.C->batch_stride(),
|
||||
gemm_workspace_.Reference->batch_stride()
|
||||
);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
results_.back().verification_map[provider] = Disposition::kNotRun;
|
||||
return true;
|
||||
}
|
||||
|
||||
results_.back().status = status;
|
||||
|
||||
if (provider == library::Provider::kReferenceHost) {
|
||||
gemm_workspace_.Reference->copy_from_host(ptr_D);
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
|
||||
results_.back().verification_map[provider] = compare_tensors(
|
||||
options,
|
||||
*gemm_workspace_.Computed,
|
||||
*gemm_workspace_.Reference,
|
||||
gemm_workspace_.Computed->batch_stride()
|
||||
);
|
||||
|
||||
// Save workspace if incorrect
|
||||
if (options.verification.save_workspace == SaveWorkspace::kIncorrect &&
|
||||
results_.back().verification_map[provider] == Disposition::kIncorrect) {
|
||||
|
||||
save_workspace(
|
||||
device_context,
|
||||
options,
|
||||
gemm_desc,
|
||||
library::Provider::kCUTLASS,
|
||||
provider);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool GemmOperationProfiler::profile(
|
||||
Options const &options,
|
||||
@@ -672,6 +887,10 @@ bool GemmOperationProfiler::profile(
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
|
||||
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
@@ -687,6 +906,100 @@ bool GemmOperationProfiler::profile(
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Method to profile a CUTLASS Operation
|
||||
Status GemmOperationProfiler::profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace) {
|
||||
|
||||
GpuTimer timer;
|
||||
|
||||
//
|
||||
// Optional sleep to limit power consumption and thermals
|
||||
//
|
||||
|
||||
sleep(options.profiling.sleep_duration);
|
||||
|
||||
//
|
||||
// Warmup loop
|
||||
//
|
||||
|
||||
Status status;
|
||||
|
||||
for (int iteration = 0; iteration < options.profiling.warmup_iterations; ++iteration) {
|
||||
|
||||
int problem_idx = (iteration % gemm_workspace_.problem_count) * problem_.batch_count;
|
||||
|
||||
gemm_workspace_.arguments.A = gemm_workspace_.A->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.B = gemm_workspace_.B->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.C = gemm_workspace_.C->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.D = gemm_workspace_.Computed->batch_data(problem_idx);
|
||||
|
||||
// Execute the CUTLASS operation
|
||||
status = operation->run(
|
||||
&gemm_workspace_.arguments,
|
||||
host_workspace,
|
||||
device_workspace);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize GPU timer
|
||||
//
|
||||
|
||||
timer.start();
|
||||
|
||||
//
|
||||
// Profiling loop
|
||||
//
|
||||
|
||||
int Iterations = options.profiling.iterations;
|
||||
|
||||
int iteration = 0;
|
||||
for (; iteration < Iterations; ++iteration) {
|
||||
|
||||
// Iterate over copies of the problem in memory
|
||||
int workspace_idx = options.profiling.warmup_iterations + iteration;
|
||||
int problem_idx = (workspace_idx % gemm_workspace_.problem_count) * problem_.batch_count;
|
||||
|
||||
gemm_workspace_.arguments.A = gemm_workspace_.A->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.B = gemm_workspace_.B->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.C = gemm_workspace_.C->batch_data(problem_idx);
|
||||
gemm_workspace_.arguments.D = gemm_workspace_.Computed->batch_data(problem_idx);
|
||||
|
||||
status = operation->run(
|
||||
arguments,
|
||||
host_workspace,
|
||||
device_workspace);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Wait for completion
|
||||
//
|
||||
|
||||
timer.stop_and_wait();
|
||||
|
||||
//
|
||||
// Update performance result
|
||||
//
|
||||
|
||||
runtime = timer.duration(iteration);
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
@@ -59,6 +59,8 @@ public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct GemmProblem {
|
||||
|
||||
cutlass::library::GemmUniversalMode mode;
|
||||
int64_t m;
|
||||
int64_t n;
|
||||
int64_t k;
|
||||
@@ -67,14 +69,15 @@ public:
|
||||
int64_t ldc;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
int split_k_slices;
|
||||
int batch_count;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
GemmProblem():
|
||||
mode(library::GemmUniversalMode::kGemm),
|
||||
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
@@ -83,6 +86,12 @@ public:
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::GemmDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::GemmDescription const &operation_desc) const;
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
@@ -99,6 +108,10 @@ public:
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Number of copies of the problem workspace which are visited sequentially during
|
||||
/// profiling to avoid camping in the last level cache.
|
||||
int problem_count;
|
||||
|
||||
library::GemmUniversalConfiguration configuration;
|
||||
library::GemmUniversalArguments arguments;
|
||||
|
||||
@@ -113,7 +126,7 @@ public:
|
||||
//
|
||||
|
||||
GemmWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr), problem_count(1) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
@@ -200,6 +213,24 @@ protected:
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against host and device references
|
||||
bool verify_with_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Method to profile a CUTLASS Operation
|
||||
Status profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -268,6 +268,7 @@ int OperationProfiler::profile_all(
|
||||
|
||||
std::string operation_name(operation->description().name);
|
||||
|
||||
// Filter kernels by name
|
||||
bool filtered_by_name = options.operation_names.empty();
|
||||
if (!filtered_by_name) {
|
||||
|
||||
@@ -279,6 +280,13 @@ int OperationProfiler::profile_all(
|
||||
}
|
||||
}
|
||||
|
||||
for (auto const & op_name : options.excluded_operation_names) {
|
||||
if (find_string_matches_(op_name, operation_name)) {
|
||||
filtered_by_name = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
|
||||
continue;
|
||||
}
|
||||
@@ -410,7 +418,8 @@ void OperationProfiler::sleep(int sleep_duration) {
|
||||
Disposition OperationProfiler::compare_tensors(
|
||||
Options const &options,
|
||||
DeviceAllocation &experimental,
|
||||
DeviceAllocation &reference) {
|
||||
DeviceAllocation &reference,
|
||||
int64_t count) {
|
||||
|
||||
if (experimental.type() != reference.type()) {
|
||||
return Disposition::kIncorrect;
|
||||
@@ -418,6 +427,10 @@ Disposition OperationProfiler::compare_tensors(
|
||||
|
||||
bool passed = false;
|
||||
|
||||
if (count == 0) {
|
||||
count = reference.capacity();
|
||||
}
|
||||
|
||||
if (options.verification.epsilon == 0) {
|
||||
|
||||
// bit-level equality
|
||||
@@ -425,7 +438,7 @@ Disposition OperationProfiler::compare_tensors(
|
||||
experimental.type(),
|
||||
experimental.data(),
|
||||
reference.data(),
|
||||
experimental.capacity());
|
||||
count);
|
||||
}
|
||||
else {
|
||||
|
||||
@@ -434,7 +447,7 @@ Disposition OperationProfiler::compare_tensors(
|
||||
experimental.type(),
|
||||
experimental.data(),
|
||||
reference.data(),
|
||||
experimental.capacity(),
|
||||
count,
|
||||
options.verification.epsilon,
|
||||
options.verification.nonzero_floor);
|
||||
}
|
||||
@@ -483,7 +496,7 @@ Status OperationProfiler::profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void const *arguments,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace) {
|
||||
|
||||
|
||||
@@ -189,7 +189,8 @@ public:
|
||||
static Disposition compare_tensors(
|
||||
Options const &options,
|
||||
DeviceAllocation &experimental,
|
||||
DeviceAllocation &reference);
|
||||
DeviceAllocation &reference,
|
||||
int64_t count = 0);
|
||||
|
||||
static void save_workspace(
|
||||
DeviceContext &device_context,
|
||||
@@ -225,7 +226,7 @@ protected:
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void const *arguments,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
|
||||
@@ -70,6 +70,16 @@ Options::Device::Device(cutlass::CommandLine const &cmdline) {
|
||||
properties.major = cc / 10;
|
||||
properties.minor = cc % 10;
|
||||
}
|
||||
|
||||
// Permit overriding the L2 cache capacity
|
||||
if (cmdline.check_cmd_line_flag("llc-capacity")) {
|
||||
int llc_capacity = 0;
|
||||
cmdline.get_cmd_line_argument("llc-capacity", llc_capacity, 0);
|
||||
|
||||
if (llc_capacity >= 0) {
|
||||
properties.l2CacheSize = (llc_capacity << 10);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -107,7 +117,12 @@ void Options::Device::print_usage(std::ostream &out) const {
|
||||
|
||||
out
|
||||
<< " --compute-capability=<int> "
|
||||
<< " Override the compute capability.\n\n";
|
||||
<< " Override the compute capability.\n\n"
|
||||
|
||||
<< " --llc-capacity=<capacity in KiB> "
|
||||
<< " Capacity of last-level cache in kilobytes. If this is non-zero," << end_of_line
|
||||
<< " profiling phases cycle through different input tensors to induce" << end_of_line
|
||||
<< " capacity misses in the L2.\n\n";
|
||||
|
||||
}
|
||||
|
||||
@@ -189,6 +204,7 @@ Options::Initialization::Initialization(cutlass::CommandLine const &cmdline) {
|
||||
// set uniform data distribution with range [-4, 4]
|
||||
data_distribution.set_uniform(-4, 4, 0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -364,7 +380,8 @@ void Options::Library::print_options(std::ostream &out, int indent) const {
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
|
||||
|
||||
|
||||
cmdline.get_cmd_line_argument("workspace-count", workspace_count, 0);
|
||||
cmdline.get_cmd_line_argument("warmup-iterations", warmup_iterations, 10);
|
||||
cmdline.get_cmd_line_argument("profiling-iterations", iterations, 100);
|
||||
cmdline.get_cmd_line_argument("sleep-duration", sleep_duration, 50);
|
||||
@@ -391,6 +408,11 @@ void Options::Profiling::print_usage(std::ostream &out) const {
|
||||
|
||||
out << "Profiling:\n"
|
||||
|
||||
<< " --workspace-count=<workspace count> "
|
||||
<< " Number of discrete workspaces maintained to avoid cache-resident " << end_of_line
|
||||
<< " If zero (default), the amount is chosen for each workload based on " << end_of_line
|
||||
<< " capacity of the last-level cache.\n\n"
|
||||
|
||||
<< " --profiling-iterations=<iterations> "
|
||||
<< " Number of iterations to profile each kernel. If zero, kernels" << end_of_line
|
||||
<< " are launched up to the profiling duration.\n\n"
|
||||
@@ -672,6 +694,10 @@ Options::Options(cutlass::CommandLine const &cmdline):
|
||||
cmdline.get_cmd_line_arguments("kernels", operation_names);
|
||||
}
|
||||
|
||||
if (cmdline.check_cmd_line_flag("ignore-kernels")) {
|
||||
cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
|
||||
}
|
||||
|
||||
// Prevent launches on the device for anything other than CUTLASS operation
|
||||
if (execution_mode == ExecutionMode::kTrace) {
|
||||
initialization.provider = library::Provider::kReferenceHost;
|
||||
@@ -706,6 +732,9 @@ void Options::print_usage(std::ostream &out) const {
|
||||
<< " Filter operations by kernel names. For example, call all kernels with" << end_of_line
|
||||
<< " (\"s1688\" and \"nt\") or (\"s844\" and \"tn\" and \"align8\") in their" << end_of_line
|
||||
<< " operation name using --kernels=\"s1688*nt, s884*tn*align8\"\n\n"
|
||||
|
||||
<< " --ignore-kernels=<string_list> "
|
||||
<< " Excludes kernels whose names match anything in this list.\n\n"
|
||||
;
|
||||
|
||||
//
|
||||
|
||||
@@ -175,6 +175,9 @@ public:
|
||||
/// Options related to profiling
|
||||
struct Profiling {
|
||||
|
||||
/// Number of workspaces to rotate through to avoid cache-resident working sets
|
||||
int workspace_count;
|
||||
|
||||
/// Number of iterations to warmup each kernel prior to profiling
|
||||
int warmup_iterations;
|
||||
|
||||
@@ -273,6 +276,10 @@ public:
|
||||
|
||||
/// Vector of operation name substrings
|
||||
std::vector<std::string> operation_names;
|
||||
|
||||
/// Vector of operation name substrings
|
||||
std::vector<std::string> excluded_operation_names;
|
||||
|
||||
|
||||
//
|
||||
// Detailed configuration options
|
||||
|
||||
@@ -263,6 +263,10 @@ std::ostream & PerformanceReport::print_csv_header_(
|
||||
<< ",OperationKind,Operation,Disposition,Status";
|
||||
|
||||
for (auto const &arg_name : argument_names_) {
|
||||
// Operand E is internal to the sparse kernel
|
||||
if (arg_name.compare("E") == 0)
|
||||
continue;
|
||||
|
||||
out << "," << arg_name;
|
||||
}
|
||||
|
||||
|
||||
@@ -577,19 +577,65 @@ void ProblemSpace::parse_(KernelArgument *arg, CommandLine const &cmdline) {
|
||||
std::vector<std::vector<std::string> > tokens;
|
||||
cmdline.get_cmd_line_argument_ranges(alias.c_str(), tokens);
|
||||
|
||||
for (auto const &range_tokens : tokens) {
|
||||
for (auto &range_tokens : tokens) {
|
||||
|
||||
if (!range_tokens.empty()) {
|
||||
Range range(lexical_cast<int64_t>(range_tokens.front()));
|
||||
|
||||
if (range_tokens.size() > 1) {
|
||||
range.last = lexical_cast<int64_t>(range_tokens.at(1));
|
||||
Range range;
|
||||
|
||||
if (range_tokens.front() == "rand") {
|
||||
range.mode = Range::Mode::kRandom;
|
||||
}
|
||||
else if (range_tokens.front() == "randlg2") {
|
||||
range.mode = Range::Mode::kRandomLog2;
|
||||
}
|
||||
|
||||
if (range_tokens.size() > 2) {
|
||||
range.increment = lexical_cast<int64_t>(range_tokens.at(2));
|
||||
}
|
||||
switch (range.mode) {
|
||||
case Range::Mode::kSequence:
|
||||
{
|
||||
range.first = lexical_cast<int64_t>(range_tokens.front());
|
||||
|
||||
if (range_tokens.size() > 1) {
|
||||
range.last = lexical_cast<int64_t>(range_tokens.at(1));
|
||||
}
|
||||
else {
|
||||
range.last = range.first;
|
||||
}
|
||||
|
||||
if (range_tokens.size() > 2) {
|
||||
range.increment = lexical_cast<int64_t>(range_tokens.at(2));
|
||||
}
|
||||
else {
|
||||
range.increment = 1;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case Range::Mode::kRandom: // fall-through
|
||||
case Range::Mode::kRandomLog2:
|
||||
{
|
||||
if (range_tokens.size() < 4) {
|
||||
throw std::runtime_error(
|
||||
"Range of mode 'rand' must have four tokens showing "
|
||||
"the minimum, maximum, and number of iterations. For example, "
|
||||
"rand:16:128:1000");
|
||||
}
|
||||
|
||||
range.minimum = lexical_cast<int64_t>(range_tokens.at(1));
|
||||
range.maximum = lexical_cast<int64_t>(range_tokens.at(2));
|
||||
range.first = 1;
|
||||
range.last = lexical_cast<int64_t>(range_tokens.at(3));
|
||||
range.increment = 1;
|
||||
|
||||
if (range_tokens.size() > 4) {
|
||||
range.divisible = lexical_cast<int64_t>(range_tokens.at(4));
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error("Unsupported range mode.");
|
||||
break;
|
||||
}
|
||||
|
||||
integer->ranges.push_back(range);
|
||||
}
|
||||
}
|
||||
@@ -713,6 +759,30 @@ bool arg_as_int(int64_t &int_value, KernelArgument::Value const *value_ptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_int(int &int_value, KernelArgument::Value const *value_ptr) {
|
||||
int64_t value64;
|
||||
bool obtained = arg_as_int(value64, value_ptr);
|
||||
if (obtained) {
|
||||
int_value = int(value64);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int
|
||||
bool arg_as_int(
|
||||
int &int_value,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
return arg_as_int(int_value, value_ptr);
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64
|
||||
bool arg_as_int(
|
||||
int64_t &int_value,
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <cstdlib>
|
||||
|
||||
// CUTLASS Utility includes
|
||||
#include "cutlass/util/command_line.h"
|
||||
@@ -307,10 +308,18 @@ struct Range {
|
||||
// Type definitions
|
||||
//
|
||||
|
||||
enum class Mode {
|
||||
kSequence,
|
||||
kRandom,
|
||||
kRandomLog2,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
struct Iterator {
|
||||
|
||||
int64_t value;
|
||||
int64_t increment;
|
||||
Range const *range;
|
||||
|
||||
//
|
||||
// Methods
|
||||
@@ -318,9 +327,10 @@ struct Range {
|
||||
|
||||
Iterator(
|
||||
int64_t value_ = 0,
|
||||
int64_t increment_ = 1
|
||||
int64_t increment_ = 1,
|
||||
Range const *range_ = nullptr
|
||||
):
|
||||
value(value_), increment(increment_) { }
|
||||
value(value_), increment(increment_), range(range_) { }
|
||||
|
||||
Iterator & operator++() {
|
||||
value += increment;
|
||||
@@ -341,7 +351,50 @@ struct Range {
|
||||
return !(*this == it);
|
||||
}
|
||||
|
||||
static int64_t round(int64_t value, int64_t divisible) {
|
||||
int64_t rem = (value % divisible);
|
||||
|
||||
// Round either up or down
|
||||
if (rem > divisible / 2) {
|
||||
value += (divisible - rem);
|
||||
}
|
||||
else {
|
||||
value -= rem;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
int64_t at() const {
|
||||
if (!range) {
|
||||
return value;
|
||||
}
|
||||
|
||||
switch (range->mode) {
|
||||
case Mode::kSequence: return value;
|
||||
|
||||
case Mode::kRandom: {
|
||||
double rnd = double(range->minimum) +
|
||||
double(std::rand()) / double(RAND_MAX) * (double(range->maximum) - double(range->minimum));
|
||||
|
||||
int64_t value = int64_t(rnd);
|
||||
|
||||
return round(value, range->divisible);
|
||||
}
|
||||
break;
|
||||
|
||||
case Mode::kRandomLog2: {
|
||||
double lg2_minimum = std::log(double(range->minimum)) / std::log(2.0);
|
||||
double lg2_maximum = std::log(double(range->maximum)) / std::log(2.0);
|
||||
double rnd = lg2_minimum + double(std::rand()) / double(RAND_MAX) * (lg2_maximum - lg2_minimum);
|
||||
|
||||
int64_t value = int64_t(std::pow(2.0, rnd));
|
||||
|
||||
return round(value, range->divisible);
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -357,20 +410,29 @@ struct Range {
|
||||
int64_t first; ///< first element in range
|
||||
int64_t last; ///< last element in range
|
||||
int64_t increment; ///< additive increment between values
|
||||
|
||||
Mode mode; ///< mode selection enables alternative values
|
||||
int64_t minimum; ///< minimum value to return
|
||||
int64_t maximum; ///< maximum value to return
|
||||
int64_t divisible; ///< rounds value down to an integer multiple of this value
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Default constructor - range acts as a scalar
|
||||
Range(int64_t first_ = 0): first(first_), last(first_), increment(1) { }
|
||||
Range(int64_t first_ = 0): first(first_), last(first_), increment(1), mode(Mode::kSequence), minimum(0), maximum(0), divisible(1) { }
|
||||
|
||||
/// Range acts as a range
|
||||
Range(
|
||||
int64_t first_,
|
||||
int64_t last_,
|
||||
int64_t increment_ = 1
|
||||
): first(first_), last(last_), increment(increment_) {
|
||||
int64_t increment_ = 1,
|
||||
Mode mode_ = Mode::kSequence,
|
||||
int64_t minimum_ = 0,
|
||||
int64_t maximum_ = 0,
|
||||
int64_t divisible_ = 1
|
||||
): first(first_), last(last_), increment(increment_), mode(mode_), minimum(minimum_), maximum(maximum_), divisible(divisible_) {
|
||||
|
||||
// Helpers to avoid constructing invalid ranges
|
||||
if (increment > 0) {
|
||||
@@ -389,14 +451,29 @@ struct Range {
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper to construct a sequence range
|
||||
static Range Sequence(int64_t first_, int64_t last_, int64_t increment_ = 1) {
|
||||
return Range(first_, last_, increment_, Mode::kSequence);
|
||||
}
|
||||
|
||||
/// Helper to construct a range that is a random distribution
|
||||
static Range Random(int64_t minimum_, int64_t maximum_, int64_t count_, int64_t divisible_ = 1) {
|
||||
return Range(1, count_, 1, Mode::kRandom, minimum_, maximum_, divisible_);
|
||||
}
|
||||
|
||||
/// Helper to construct a range that is a random distribution over a log scale
|
||||
static Range RandomLog2(int64_t minimum_, int64_t maximum_, int64_t count_, int64_t divisible_ = 1) {
|
||||
return Range(1, count_, 1, Mode::kRandomLog2, minimum_, maximum_, divisible_);
|
||||
}
|
||||
|
||||
/// Returns an iterator to the first element within the range
|
||||
Iterator begin() const {
|
||||
return Iterator(first, increment);
|
||||
return Iterator(first, increment, this);
|
||||
}
|
||||
|
||||
/// Returns an iterator to the first element *after* the range
|
||||
Iterator end() const {
|
||||
return Iterator(first + ((last - first)/increment + 1) * increment, increment);
|
||||
return Iterator(first + ((last - first)/increment + 1) * increment, increment, this);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -770,9 +847,19 @@ private:
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Lexically casts an argument to an int if it is defined. Returns true if not null.
|
||||
bool arg_as_int(int &int_value, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_int(int64_t &int_value, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_int(
|
||||
int &int_value,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_int(
|
||||
int64_t &int_value,
|
||||
|
||||
@@ -0,0 +1,560 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Execution environment
|
||||
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
#include <ios>
|
||||
|
||||
#include "cublas_helpers.h"
|
||||
#include "sparse_gemm_operation_profiler.h"
|
||||
#include "gpu_timer.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Ctor
|
||||
SparseGemmOperationProfiler::SparseGemmOperationProfiler(Options const &options):
|
||||
OperationProfiler(
|
||||
options,
|
||||
library::OperationKind::kSparseGemm,
|
||||
{
|
||||
{ArgumentTypeID::kEnumerated, {"gemm_kind"}, "Variant of GEMM (e.g. gemm, planar complex, batched, ...)"},
|
||||
{ArgumentTypeID::kInteger, {"m", "problem-size::m"}, "M dimension of the GEMM problem space"},
|
||||
{ArgumentTypeID::kInteger, {"n", "problem-size::n"}, "N dimension of the GEMM problem space"},
|
||||
{ArgumentTypeID::kInteger, {"k", "problem-size::k"}, "K dimension of the GEMM problem space"},
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
|
||||
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
|
||||
{ArgumentTypeID::kTensor, {"E"}, "Tensor storing the E operand"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kInteger, {"split_k_slices"}, "Number of partitions of K dimension"},
|
||||
{ArgumentTypeID::kInteger, {"batch_count"}, "Number of GEMMs computed in one batch"},
|
||||
}
|
||||
) {
|
||||
|
||||
description_ = " Structured sparse GEMM. D = alpha * A*B + beta * C";
|
||||
}
|
||||
|
||||
/// Destructor
|
||||
SparseGemmOperationProfiler::~SparseGemmOperationProfiler() {
|
||||
|
||||
}
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
void SparseGemmOperationProfiler::print_usage(std::ostream &out) const {
|
||||
out << "Sparse GEMM" << "\n\n";
|
||||
|
||||
OperationProfiler::print_usage(out);
|
||||
}
|
||||
|
||||
/// Prints examples
|
||||
void SparseGemmOperationProfiler::print_examples(std::ostream &out) const {
|
||||
|
||||
out << "\nExamples:\n\n"
|
||||
<< "Profile a particular problem size:\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --m=1024 --n=1024 --k=128\n\n"
|
||||
|
||||
<< "Schmoo over problem size and beta:\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --m=1024:4096:256 --n=1024:4096:256 --k=128:8192:128 --beta=0,1,2.5\n\n"
|
||||
|
||||
<< "Schmoo over accumulator types:\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --accumulator-type=f16,f32\n\n"
|
||||
|
||||
<< "Run when A is f16 with column-major and B is any datatype with row-major (For column major, use column, col, or n. For row major use, row or t):\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --A=f16:column --B=*:row\n\n"
|
||||
|
||||
<< "Using various input value distribution:\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --dist=uniform,min:0,max:3\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --dist=gaussian,mean:0,stddev:3\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --dist=sequential,start:0,delta:1\n\n"
|
||||
|
||||
<< "Run a kernel with cta tile size of 256x128x32 and save workspace if results are incorrect (note that --cta-tile::k=32 is default cta-tile size):\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm --cta_m=256 --cta_n=128 --cta_k=32 --save-workspace=incorrect\n\n"
|
||||
|
||||
<< "Test your changes to gemm kernels with a quick functional test and save results in functional-test.csv:\n"
|
||||
<< " $ cutlass_profiler --operation=SparseGemm \\ \n"
|
||||
<< " --m=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --n=8,56,120,136,256,264,512,520,1024,1032,4096,8192,16384 \\ \n"
|
||||
<< " --k=8,16,32,64,128,256,288,384,504,512,520 \\ \n"
|
||||
<< " --beta=0,1,2 --profiling-iterations=1 \\ \n"
|
||||
<< " --providers=cutlass --output=functional-test.csv\n\n";
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Status SparseGemmOperationProfiler::SparseGemmProblem::parse(
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!arg_as_int(this->m, "m", problem_space, problem)) {
|
||||
// default value
|
||||
this->m = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->n, "n", problem_space, problem)) {
|
||||
// default value
|
||||
this->n = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->k, "k", problem_space, problem)) {
|
||||
// default value
|
||||
this->k = 1024;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->split_k_slices, "split_k_slices", problem_space, problem)) {
|
||||
// default value
|
||||
this->split_k_slices = 1;
|
||||
}
|
||||
|
||||
if (!arg_as_int(this->batch_count, "batch_count", problem_space, problem)) {
|
||||
// default value
|
||||
this->batch_count = 1;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.A, "A", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.B, "B", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.C, "C", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.E, "E", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
"alpha",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->alpha, operation_desc.element_epilogue, 1)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->beta,
|
||||
operation_desc.element_epilogue,
|
||||
"beta",
|
||||
problem_space,
|
||||
problem)) {
|
||||
|
||||
if (!cast_from_double(this->beta, operation_desc.element_epilogue, 0)) {
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
this->elements_per_128b =
|
||||
128 / library::sizeof_bits(operation_desc.A.element);
|
||||
|
||||
this->lda = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.A.layout,
|
||||
{int(this->m), int(this->k) / int(this->sparse)})
|
||||
.front();
|
||||
|
||||
this->ldb = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.B.layout, {int(this->k), int(this->n)}).front();
|
||||
|
||||
this->ldc = DeviceAllocation::get_packed_layout(
|
||||
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
|
||||
|
||||
this->lde =
|
||||
DeviceAllocation::get_packed_layout(
|
||||
operation_desc.E.layout,
|
||||
{int(this->m), int(this->k / this->sparse / this->elements_per_128b)})
|
||||
.front();
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Initializes a performance result
|
||||
void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.arguments.resize(problem_space.rank());
|
||||
|
||||
set_argument(result, "gemm_kind", problem_space, library::to_string(operation_desc.gemm_kind));
|
||||
|
||||
set_argument(result, "A", problem_space,
|
||||
std::string(library::to_string(operation_desc.A.element)) + ":" + library::to_string(operation_desc.A.layout));
|
||||
|
||||
set_argument(result, "B", problem_space,
|
||||
std::string(library::to_string(operation_desc.B.element)) + ":" + library::to_string(operation_desc.B.layout));
|
||||
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
|
||||
set_argument(result, "split_k_slices", problem_space, split_k_slices);
|
||||
set_argument(result, "batch_count", problem_space, batch_count);
|
||||
|
||||
set_argument(result, "alpha", problem_space,
|
||||
library::lexical_cast(alpha, operation_desc.element_epilogue));
|
||||
|
||||
set_argument(result, "beta", problem_space,
|
||||
library::lexical_cast(beta, operation_desc.element_epilogue));
|
||||
}
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
Status SparseGemmOperationProfiler::initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::SparseGemmDescription const &operation_desc =
|
||||
static_cast<library::SparseGemmDescription const &>(operation->description());
|
||||
|
||||
if (operation_desc.gemm_kind != library::GemmKind::kSparse) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
Status status = problem_.parse(operation_desc, problem_space, problem);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
gemm_workspace_.configuration.problem_size.m() = int(problem_.m);
|
||||
gemm_workspace_.configuration.problem_size.n() = int(problem_.n);
|
||||
gemm_workspace_.configuration.problem_size.k() = int(problem_.k);
|
||||
gemm_workspace_.configuration.lda = problem_.lda;
|
||||
gemm_workspace_.configuration.ldb = problem_.ldb;
|
||||
gemm_workspace_.configuration.ldc = problem_.ldc;
|
||||
gemm_workspace_.configuration.ldd = problem_.ldc;
|
||||
gemm_workspace_.configuration.lde = problem_.lde;
|
||||
|
||||
gemm_workspace_.arguments.A = nullptr;
|
||||
gemm_workspace_.arguments.B = nullptr;
|
||||
gemm_workspace_.arguments.C = nullptr;
|
||||
gemm_workspace_.arguments.D = nullptr;
|
||||
gemm_workspace_.arguments.E = nullptr;
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
initialize_result_(this->model_result_, options, operation_desc, problem_space);
|
||||
|
||||
return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
|
||||
}
|
||||
|
||||
/// Initializes the performance result
|
||||
void SparseGemmOperationProfiler::initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space) {
|
||||
|
||||
result.provider = library::Provider::kCUTLASS;
|
||||
result.disposition = Disposition::kNotRun;
|
||||
result.status = Status::kSuccess;
|
||||
result.operation_name = operation_desc.name;
|
||||
|
||||
problem_.initialize_result(result, operation_desc, problem_space);
|
||||
|
||||
OperationProfiler::initialize_result_(result, operation_desc, problem_space);
|
||||
|
||||
// Input bytes read and Output bytes written for the gemm problem
|
||||
result.bytes =
|
||||
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_.m / 8) *
|
||||
problem_.k / problem_.sparse +
|
||||
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_.n / 8) *
|
||||
problem_.k +
|
||||
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) *
|
||||
problem_.n +
|
||||
int64_t(library::sizeof_bits(operation_desc.E.element) * problem_.m / 8) *
|
||||
problem_.k / problem_.sparse / problem_.elements_per_128b;
|
||||
|
||||
// Set is_beta_zero true if beta is zero
|
||||
bool is_beta_zero = std::all_of(problem_.beta.begin(), problem_.beta.end(), [](uint8_t i) { return i==0; });
|
||||
|
||||
// Output bytes read for the gemm problem for non-zero beta values
|
||||
if (!is_beta_zero) {
|
||||
result.bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_.m / 8) * problem_.n;
|
||||
}
|
||||
|
||||
result.flops = 2 * (problem_.m * problem_.n * problem_.k + problem_.m * problem_.n);
|
||||
result.runtime = 0;
|
||||
|
||||
}
|
||||
|
||||
/// Initializes workspace
|
||||
Status SparseGemmOperationProfiler::initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
library::SparseGemmDescription const &operation_desc =
|
||||
static_cast<library::SparseGemmDescription const &>(operation->description());
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
gemm_workspace_.A = device_context.allocate_tensor(
|
||||
options,
|
||||
"A",
|
||||
operation_desc.A.element,
|
||||
operation_desc.A.layout,
|
||||
{int(problem_.m), int(problem_.k) / int(problem_.sparse)},
|
||||
{int(problem_.lda)}
|
||||
);
|
||||
|
||||
gemm_workspace_.B = device_context.allocate_tensor(
|
||||
options,
|
||||
"B",
|
||||
operation_desc.B.element,
|
||||
operation_desc.B.layout,
|
||||
{int(problem_.k), int(problem_.n)},
|
||||
{int(problem_.ldb)}
|
||||
);
|
||||
|
||||
gemm_workspace_.C = device_context.allocate_tensor(
|
||||
options,
|
||||
"C",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
gemm_workspace_.Computed = device_context.allocate_tensor(
|
||||
"D",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
gemm_workspace_.E = device_context.allocate_sparsemeta_tensor(
|
||||
options,
|
||||
"E",
|
||||
operation_desc.E.element,
|
||||
operation_desc.E.layout,
|
||||
operation_desc.A.element,
|
||||
{int(problem_.m), int(problem_.k) / int(problem_.sparse) / int(problem_.elements_per_128b)},
|
||||
{int(problem_.lde)}
|
||||
);
|
||||
|
||||
gemm_workspace_.Reference = device_context.allocate_tensor(
|
||||
"Reference",
|
||||
operation_desc.C.element,
|
||||
operation_desc.C.layout,
|
||||
{int(problem_.m), int(problem_.n)},
|
||||
{int(problem_.ldc)}
|
||||
);
|
||||
|
||||
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
|
||||
}
|
||||
|
||||
//
|
||||
// Initialize the CUTLASS operation
|
||||
//
|
||||
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
if (options.execution_mode != ExecutionMode::kDryRun) {
|
||||
|
||||
uint64_t workspace_size = operation->get_host_workspace_size(&gemm_workspace_.configuration);
|
||||
gemm_workspace_.host_workspace.resize(workspace_size, 0);
|
||||
|
||||
workspace_size = operation->get_device_workspace_size(&gemm_workspace_.configuration);
|
||||
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
|
||||
|
||||
status = operation->initialize(
|
||||
&gemm_workspace_.configuration,
|
||||
gemm_workspace_.host_workspace.data(),
|
||||
gemm_workspace_.device_workspace.data());
|
||||
}
|
||||
|
||||
//
|
||||
// If CUTLASS is enabled, generate a result for it
|
||||
//
|
||||
|
||||
results_.push_back(model_result_);
|
||||
results_.back().provider = library::Provider::kCUTLASS;
|
||||
results_.back().op_kind = library::OperationKind::kSparseGemm;
|
||||
results_.back().disposition = Disposition::kNotRun;
|
||||
|
||||
for(auto &verification_provider : options.verification.providers) {
|
||||
results_.back().verification_map[verification_provider] = Disposition::kNotRun;
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
bool SparseGemmOperationProfiler::verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (!options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.execution_mode == ExecutionMode::kDryRun) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize structure containing GEMM arguments
|
||||
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
|
||||
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
|
||||
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
|
||||
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
|
||||
gemm_workspace_.arguments.E = gemm_workspace_.E->data();
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
//
|
||||
// Run the CUTLASS operation
|
||||
//
|
||||
|
||||
results_.back().status = operation->run(
|
||||
&gemm_workspace_.arguments,
|
||||
gemm_workspace_.host_workspace.data(),
|
||||
gemm_workspace_.device_workspace.data());
|
||||
|
||||
if (results_.back().status != Status::kSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
results_.back().disposition = Disposition::kFailed;
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUTLASS op ran the but not yet verified against any verification provider
|
||||
results_.back().disposition = Disposition::kNotVerified;
|
||||
|
||||
//
|
||||
// Run verification providers
|
||||
//
|
||||
|
||||
if (options.verification.enabled) {
|
||||
|
||||
// Update disposition to worst case verification outcome among all
|
||||
// verification providers which are supported
|
||||
bool is_any_verification_run_passed = false;
|
||||
|
||||
for(auto &m : results_.back().verification_map) {
|
||||
if(m.second == Disposition::kFailed || m.second == Disposition::kIncorrect) {
|
||||
results_.back().disposition = m.second;
|
||||
return true;
|
||||
}
|
||||
if(!is_any_verification_run_passed && m.second == Disposition::kPassed) {
|
||||
is_any_verification_run_passed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(is_any_verification_run_passed) {
|
||||
results_.back().disposition = Disposition::kPassed;
|
||||
}
|
||||
}
|
||||
|
||||
// Return true means continue profiling
|
||||
return true;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Measures performance results
|
||||
bool SparseGemmOperationProfiler::profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
// Initialize structure containing GEMM arguments
|
||||
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
|
||||
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
|
||||
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
|
||||
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
|
||||
gemm_workspace_.arguments.E = gemm_workspace_.E->data();
|
||||
gemm_workspace_.arguments.alpha = problem_.alpha.data();
|
||||
gemm_workspace_.arguments.beta = problem_.beta.data();
|
||||
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
|
||||
|
||||
results_.back().status = profile_cutlass_(
|
||||
results_.back().runtime,
|
||||
options,
|
||||
operation,
|
||||
&gemm_workspace_.arguments,
|
||||
gemm_workspace_.host_workspace.data(),
|
||||
gemm_workspace_.device_workspace.data()
|
||||
);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,208 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#include "gemm_operation_profiler.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class SparseGemmOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct SparseGemmProblem {
|
||||
int64_t m;
|
||||
int64_t n;
|
||||
int64_t k;
|
||||
int64_t lda;
|
||||
int64_t ldb;
|
||||
int64_t ldc;
|
||||
int64_t lde;
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
int64_t split_k_slices;
|
||||
int64_t batch_count;
|
||||
static int const sparse = 2;
|
||||
// every 128b ElementA uses one elementE
|
||||
int elements_per_128b;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
SparseGemmProblem():
|
||||
m(16), n(16), k(16), lda(0), ldb(0), ldc(0), lde(0), split_k_slices(1), batch_count(1) { }
|
||||
|
||||
/// Parses the problem
|
||||
Status parse(
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes a performance result
|
||||
void initialize_result(
|
||||
PerformanceResult &result,
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct SparseGemmWorkspace {
|
||||
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *E;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
library::SparseGemmConfiguration configuration;
|
||||
library::SparseGemmArguments arguments;
|
||||
|
||||
/// Buffer used for the operation's host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
SparseGemmWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), E(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
// GEMM problem
|
||||
SparseGemmProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
SparseGemmWorkspace gemm_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
SparseGemmOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~SparseGemmOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::SparseGemmDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Reference in New Issue
Block a user