leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

CUTLASS 2.6 (#298)

Browse Source 
CUTLASS 2.6
This commit is contained in:
Manish Gupta
2021-07-23 00:40:53 -04:00
committed by GitHub
parent 6c29fe20ba
commit e5d51840e8
308 changed files with 32408 additions and 4722 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/src/gemm_operation.h
+35 -31
View File
@@ -58,6 +58,8 @@ public:
using LayoutB = typename Operator::LayoutB;
using ElementC = typename Operator::ElementC;
using LayoutC = typename Operator::LayoutC;
// assuming all tensors use same type for StrideIndex
using StrideIndex = typename Operator::LayoutA::Index;
using ElementAccumulator = typename Operator::ElementAccumulator;
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
@@ -102,7 +104,7 @@ public:
OpcodeClassMap<typename Operator::OperatorClass>::kId;
description_.tile_description.math_instruction.math_operation =
MathOperationMap<typename Operator::Operator>::kId;
MathOperationMap<typename Operator::MathOperator>::kId;
description_.tile_description.minimum_compute_capability =
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMin;
@@ -141,7 +143,6 @@ public:
using LayoutC = typename Operator::LayoutC;
using ElementAccumulator = typename Operator::ElementAccumulator;
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
using OperatorArguments = typename Operator::Arguments;
public:
@@ -160,10 +161,12 @@ protected:
GemmConfiguration const *configuration) {
operator_args.problem_size = configuration->problem_size;
operator_args.ref_A = {nullptr, int(configuration->lda)};
operator_args.ref_B = {nullptr, int(configuration->ldb)};
operator_args.ref_C = {nullptr, int(configuration->ldc)};
operator_args.ref_D = {nullptr, int(configuration->ldd)};
operator_args.ref_A = {nullptr, configuration->lda};
operator_args.ref_B = {nullptr, configuration->ldb};
operator_args.ref_C = {nullptr, configuration->ldc};
operator_args.ref_D = {nullptr, configuration->ldd};
operator_args.split_k_slices = configuration->split_k_slices;
return Status::kSuccess;
@@ -360,11 +363,11 @@ protected:
SparseGemmConfiguration const *configuration) {
operator_args.problem_size = configuration->problem_size;
operator_args.ref_A = {nullptr, int(configuration->lda)};
operator_args.ref_B = {nullptr, int(configuration->ldb)};
operator_args.ref_C = {nullptr, int(configuration->ldc)};
operator_args.ref_D = {nullptr, int(configuration->ldd)};
operator_args.ref_E = {nullptr, int(configuration->lde)};
operator_args.ref_A = {nullptr, configuration->lda};
operator_args.ref_B = {nullptr, configuration->ldb};
operator_args.ref_C = {nullptr, configuration->ldc};
operator_args.ref_D = {nullptr, configuration->ldd};
operator_args.ref_E = {nullptr, configuration->lde};
return Status::kSuccess;
}
@@ -562,10 +565,10 @@ protected:
operator_args.problem_size = configuration->problem_size;
operator_args.batch_count = configuration->batch_count;
operator_args.lda = int(configuration->lda);
operator_args.ldb = int(configuration->ldb);
operator_args.ldc = int(configuration->ldc);
operator_args.ldd = int(configuration->ldd);
operator_args.lda = (configuration->lda);
operator_args.ldb = (configuration->ldb);
operator_args.ldc = (configuration->ldc);
operator_args.ldd = (configuration->ldd);
return Status::kSuccess;
}
@@ -755,14 +758,15 @@ protected:
operator_args.problem_size = configuration->problem_size;
operator_args.batch_count = configuration->batch_count;
operator_args.lda_real = int(configuration->lda_real);
operator_args.lda_imag = int(configuration->lda_imag);
operator_args.ldb_real = int(configuration->ldb_real);
operator_args.ldb_imag = int(configuration->ldb_imag);
operator_args.ldc_real = int(configuration->ldc_real);
operator_args.ldc_imag = int(configuration->ldc_imag);
operator_args.ldd_real = int(configuration->ldd_real);
operator_args.ldd_imag = int(configuration->ldd_imag);
operator_args.lda_real = configuration->lda_real;
operator_args.lda_imag = configuration->lda_imag;
operator_args.ldb_real = configuration->ldb_real;
operator_args.ldb_imag = configuration->ldb_imag;
operator_args.ldc_real = configuration->ldc_real;
operator_args.ldc_imag = configuration->ldc_imag;
operator_args.ldd_real = configuration->ldd_real;
operator_args.ldd_imag = configuration->ldd_imag;
return Status::kSuccess;
}
@@ -960,14 +964,14 @@ protected:
operator_args.problem_size = configuration->problem_size;
operator_args.batch_count = configuration->batch_count;
operator_args.lda_real = int(configuration->lda_real);
operator_args.lda_imag = int(configuration->lda_imag);
operator_args.ldb_real = int(configuration->ldb_real);
operator_args.ldb_imag = int(configuration->ldb_imag);
operator_args.ldc_real = int(configuration->ldc_real);
operator_args.ldc_imag = int(configuration->ldc_imag);
operator_args.ldd_real = int(configuration->ldd_real);
operator_args.ldd_imag = int(configuration->ldd_imag);
operator_args.lda_real = configuration->lda_real;
operator_args.lda_imag = configuration->lda_imag;
operator_args.ldb_real = configuration->ldb_real;
operator_args.ldb_imag = configuration->ldb_imag;
operator_args.ldc_real = configuration->ldc_real;
operator_args.ldc_imag = configuration->ldc_imag;
operator_args.ldd_real = configuration->ldd_real;
operator_args.ldd_imag = configuration->ldd_imag;
return Status::kSuccess;
}
tools/library/src/handle.cu
+28 -28
View File
@@ -204,18 +204,18 @@ static int gemm_problem_alignment(
int K,
NumericTypeID element_A,
void const *ptr_A,
int lda,
int64_t lda,
int64_t batch_stride_A,
NumericTypeID element_B,
void const *ptr_B,
int ldb,
int64_t ldb,
int64_t batch_stride_B,
NumericTypeID element_C,
void const * ptr_C,
int ldc,
int64_t ldc,
int64_t batch_stride_C,
void const * ptr_D,
int ldd,
int64_t ldd,
int64_t batch_stride_D,
int max_alignment_in_bytes = 16
) {
@@ -338,24 +338,24 @@ Status Handle::gemm(
ComplexTransform transform_A, /// Complex transformation applied to A matrix - ignored for real-valued matrices
void const * ptr_A, /// Pointer to A matrix in Global Memory
int lda, /// Leading dimension of A matrix
int64_t lda, /// Leading dimension of A matrix
NumericTypeID element_B, /// Data type of B matrix elements
LayoutTypeID layout_B, /// Layout of B matrix
ComplexTransform transform_B, /// Complex transformation applied to B matrix - ignored for real-valued matrices
void const * ptr_B, /// Pointer to B matrix in Global Memory
int ldb, /// Leading dimension of B matrix
int64_t ldb, /// Leading dimension of B matrix
void const * beta, /// Pointer to beta scalar
NumericTypeID element_C, /// Data type of C and D matrices
void const * ptr_C, /// Pointer to C matrix
int ldc, /// Leading dimension of C matrix
int64_t ldc, /// Leading dimension of C matrix
void * ptr_D, /// Pointer to D matrix
int ldd /// Leading dimension of D matrix
int64_t ldd /// Leading dimension of D matrix
) {
//
@@ -494,24 +494,24 @@ Status Handle::gemm_universal(
ComplexTransform transform_A, /// Complex transformation applied to A matrix - ignored for real-valued matrices
void const * ptr_A, /// Pointer to A matrix in Global Memory
int lda, /// Leading dimension of A matrix
int64_t lda, /// Leading dimension of A matrix
NumericTypeID element_B, /// Data type of B matrix elements
LayoutTypeID layout_B, /// Layout of B matrix
ComplexTransform transform_B, /// Complex transformation applied to B matrix - ignored for real-valued matrices
void const * ptr_B, /// Pointer to B matrix in Global Memory
int ldb, /// Leading dimension of B matrix
int64_t ldb, /// Leading dimension of B matrix
void const * beta, /// Pointer to beta scalar
NumericTypeID element_C, /// Data type of C and D matrices
void const * ptr_C, /// Pointer to C matrix
int ldc, /// Leading dimension of C matrix
int64_t ldc, /// Leading dimension of C matrix
void * ptr_D, /// Pointer to D matrix
int ldd, /// Leading dimension of D matrix
int64_t ldd, /// Leading dimension of D matrix
int batch_count, /// Batch count or number of split-K slices
@@ -672,8 +672,8 @@ Status Handle::gemm_planar_complex(
void const * ptr_A_real, /// Pointer to real part of A matrix
void const * ptr_A_imag, /// Pointer to imaginary part of A matrix
int lda_real, /// Leading dimension of real part of A matrix
int lda_imag, /// Leading dimension of imaginary part of A matrix
int64_t lda_real, /// Leading dimension of real part of A matrix
int64_t lda_imag, /// Leading dimension of imaginary part of A matrix
NumericTypeID element_B, /// Data type of B matrix elements
LayoutTypeID layout_B, /// Layout of B matrix
@@ -681,8 +681,8 @@ Status Handle::gemm_planar_complex(
void const * ptr_B_real, /// Pointer to real part of B matrix
void const * ptr_B_imag, /// Pointer to imaginary part of B matrix
int ldb_real, /// Leading dimension of real part of B matrix
int ldb_imag, /// Leading dimension of imaginary part of B matrix
int64_t ldb_real, /// Leading dimension of real part of B matrix
int64_t ldb_imag, /// Leading dimension of imaginary part of B matrix
void const * beta, /// Pointer to beta scalar
@@ -690,13 +690,13 @@ Status Handle::gemm_planar_complex(
void const * ptr_C_real, /// Pointer to real part of C matrix
void const * ptr_C_imag, /// Pointer to imaginary part of C matrix
int ldc_real, /// Leading dimension of real part of C matrix
int ldc_imag, /// Leading dimension of imaginary part of C matrix
int64_t ldc_real, /// Leading dimension of real part of C matrix
int64_t ldc_imag, /// Leading dimension of imaginary part of C matrix
void * ptr_D_real, /// Pointer to real part of D matrix
void * ptr_D_imag, /// Pointer to imaginary part of D matrix
int ldd_real, /// Leading dimension of real part of D matrix
int ldd_imag, /// Leading dimension of imaginary part of D matrix
int64_t ldd_real, /// Leading dimension of real part of D matrix
int64_t ldd_imag, /// Leading dimension of imaginary part of D matrix
int batch_count, /// Number of batched GEMMs to execute
@@ -877,8 +877,8 @@ Status Handle::gemm_planar_complex_array(
void const * const * ptr_A_real, /// Pointer to array containing pointers to real part of A matrices
void const * const * ptr_A_imag, /// Pointer to array containing pointers to imaginary part of A matrices
int lda_real, /// Leading dimension of real part of A matrix
int lda_imag, /// Leading dimension of imaginary part of A matrix
int64_t lda_real, /// Leading dimension of real part of A matrix
int64_t lda_imag, /// Leading dimension of imaginary part of A matrix
NumericTypeID element_B, /// Data type of B matrix elements
LayoutTypeID layout_B, /// Layout of B matrix
@@ -887,8 +887,8 @@ Status Handle::gemm_planar_complex_array(
void const * const * ptr_B_real, /// Pointer to array containing pointers to real part of B matrices
void const * const * ptr_B_imag, /// Pointer to array containing pointers to imaginary part of B matrices
int ldb_real, /// Leading dimension of real part of B matrix
int ldb_imag, /// Leading dimension of imaginary part of B matrix
int64_t ldb_real, /// Leading dimension of real part of B matrix
int64_t ldb_imag, /// Leading dimension of imaginary part of B matrix
void const * beta, /// Pointer to beta scalar
@@ -897,14 +897,14 @@ Status Handle::gemm_planar_complex_array(
void const * const * ptr_C_real, /// Pointer to array containing pointers to real part of C matrices
void const * const * ptr_C_imag, /// Pointer to array containing poitners to imaginary part of C matrices
int ldc_real, /// Leading dimension of real part of C matrix
int ldc_imag, /// Leading dimension of imaginary part of C matrix
int64_t ldc_real, /// Leading dimension of real part of C matrix
int64_t ldc_imag, /// Leading dimension of imaginary part of C matrix
void * const * ptr_D_real, /// Pointer to array containing pointers to real part of D matrices
void * const * ptr_D_imag, /// Pointer to array containing poitners to imaginary part of D matrices
int ldd_real, /// Leading dimension of real part of D matrix
int ldd_imag /// Leading dimension of imaginary part of D matrix
int64_t ldd_real, /// Leading dimension of real part of D matrix
int64_t ldd_imag /// Leading dimension of imaginary part of D matrix
) {
//
tools/library/src/reference/conv_reference_operation.h
+9 -3
View File
@@ -115,13 +115,19 @@ struct ConvReferenceDispatcher<
layout::TensorNHWC layout_c;
layout_a.stride() =
make_Coord(config.stride_a[0], config.stride_a[1], config.stride_a[2]);
make_Coord(int32_t(config.stride_a[0]),
int32_t(config.stride_a[1]),
int32_t(config.stride_a[2]));
layout_b.stride() =
make_Coord(config.stride_b[0], config.stride_b[1], config.stride_b[2]);
make_Coord(int32_t(config.stride_b[0]),
int32_t(config.stride_b[1]),
int32_t(config.stride_b[2]));
layout_c.stride() =
make_Coord(config.stride_c[0], config.stride_c[1], config.stride_c[2]);
make_Coord(int32_t(config.stride_c[0]),
int32_t(config.stride_c[1]),
int32_t(config.stride_c[2]));
if (kProvider == Provider::kReferenceHost) {