CUTLASS 2.4 (Implicit GEMM convolution) (#147)
CUTLASS 2.4 (Implicit GEMM Convolution) Co-authored-by: Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
This commit is contained in:
Manish Gupta
co-authored by
Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
parent
c2b80ad4e4
commit
6615010cd0
@@ -26,6 +26,12 @@ if (CUTLASS_ENABLE_LIBRARY)
|
||||
add_subdirectory(library)
|
||||
endif()
|
||||
if (CUTLASS_ENABLE_PROFILER)
|
||||
add_subdirectory(profiler)
|
||||
if (NOT CUTLASS_ENABLE_LIBRARY)
|
||||
message(SEND_ERROR "Build conflict: The CUTLASS profiler requires the CUTLASS library.")
|
||||
message(SEND_ERROR " CUTLASS_ENABLE_PROFILER = ${CUTLASS_ENABLE_PROFILER}")
|
||||
message(SEND_ERROR " CUTLASS_ENABLE_LIBRARY = ${CUTLASS_ENABLE_LIBRARY}")
|
||||
else()
|
||||
add_subdirectory(profiler)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@@ -63,6 +63,15 @@ cutlass_add_library(
|
||||
src/reference/gemm.cu
|
||||
src/reference/initialize_reference_operations.cu
|
||||
|
||||
|
||||
# cutlass reduction instances in cutlass library
|
||||
src/reduction/reduction_device.cu
|
||||
src/reduction/init_reduction_operations.cu
|
||||
|
||||
# cutlass conv reference instances in cutlass library
|
||||
src/reference/conv2d.cu
|
||||
src/reference/conv3d.cu
|
||||
|
||||
)
|
||||
|
||||
file(GLOB_RECURSE GENERATOR_PYTHON_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/scripts/*.py)
|
||||
@@ -136,7 +145,7 @@ function(cutlass_add_cutlass_library)
|
||||
cutlass_library_includes
|
||||
)
|
||||
|
||||
set_target_properties(${__NAME} PROPERTIES DEBUG_POSTFIX ${CUTLASS_LIBRARY_DEBUG_POSTFIX})
|
||||
set_target_properties(${__NAME} PROPERTIES DEBUG_POSTFIX "${CUTLASS_LIBRARY_DEBUG_POSTFIX}")
|
||||
|
||||
set(OUTPUT_NAME cutlass)
|
||||
|
||||
|
||||
@@ -335,6 +335,10 @@ public:
|
||||
using HandlePtr = std::unique_ptr<Handle>;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Finds conv2d operation instances with Conv2d::ElementC = Reduction::ElementWorkspace
|
||||
Operation const* find_conv_operation_for_parallel_reduction(Operation const *operation);
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
@@ -53,6 +53,10 @@
|
||||
#include "cutlass/layout/tensor.h"
|
||||
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/conv/convolution.h"
|
||||
#include "cutlass/conv/conv2d_problem_size.h"
|
||||
#include "cutlass/conv/conv3d_problem_size.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
@@ -79,6 +83,10 @@ enum class LayoutTypeID {
|
||||
kTensorNCDHW,
|
||||
kTensorNHWC,
|
||||
kTensorNDHWC,
|
||||
kTensorNC32HW32,
|
||||
kTensorC32RSK32,
|
||||
kTensorNC64HW64,
|
||||
kTensorC64RSK64,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
@@ -138,6 +146,7 @@ enum class Provider {
|
||||
kReferenceHost,
|
||||
kReferenceDevice,
|
||||
kCUBLAS,
|
||||
kCUDNN,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
@@ -146,6 +155,8 @@ enum class Provider {
|
||||
/// Enumeration indicating the kind of operation
|
||||
enum class OperationKind {
|
||||
kGemm,
|
||||
kConv2d,
|
||||
kConv3d,
|
||||
kEqGemm,
|
||||
kSparseGemm,
|
||||
kReduction,
|
||||
@@ -204,6 +215,30 @@ enum class GemmKind {
|
||||
/// Mode of Universal GEMM
|
||||
using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
|
||||
|
||||
/// Enumeration indicating what kind of Conv2d operation to perform
|
||||
enum class ConvKind {
|
||||
kUnknown,
|
||||
kFprop,
|
||||
kDgrad,
|
||||
kWgrad,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
enum class ConvModeID {
|
||||
kCrossCorrelation,
|
||||
kConvolution,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
// Iterator algorithm enum in order of general performance-efficiency
|
||||
enum class IteratorAlgorithmID {
|
||||
kNone,
|
||||
kAnalytic,
|
||||
kOptimized,
|
||||
kInvalid
|
||||
};
|
||||
|
||||
|
||||
enum class EpilogueKind {
|
||||
kUnknown,
|
||||
kConversion,
|
||||
@@ -477,6 +512,66 @@ struct ReductionDescription : public OperationDescription {
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Description of all Conv2d operations
|
||||
struct ConvDescription : public OperationDescription {
|
||||
/// Describes the convolution dimension support (2D or 3D)
|
||||
int conv_dim;
|
||||
|
||||
/// Describes the kind of convolution
|
||||
ConvKind conv_kind;
|
||||
|
||||
/// Describes the type of iterator algorithm (analytic or precomputed)
|
||||
IteratorAlgorithmID iterator_algorithm;
|
||||
|
||||
/// Describes the A operand
|
||||
TensorDescription A;
|
||||
|
||||
/// Describes the B operand
|
||||
TensorDescription B;
|
||||
|
||||
/// Describes the C operand
|
||||
TensorDescription C;
|
||||
|
||||
/// Describes the data type of the scalars passed to the epilogue
|
||||
NumericTypeID element_epilogue;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
// Returns Activation TensorDescription
|
||||
TensorDescription activation() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return A;
|
||||
case library::ConvKind::kDgrad : return C;
|
||||
case library::ConvKind::kWgrad : return B;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Filter TensorDescription
|
||||
TensorDescription filter() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return B;
|
||||
case library::ConvKind::kDgrad : return B;
|
||||
case library::ConvKind::kWgrad : return C;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Output TensorDescription
|
||||
TensorDescription output() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return C;
|
||||
case library::ConvKind::kDgrad : return A;
|
||||
case library::ConvKind::kWgrad : return A;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Base class for all operations
|
||||
@@ -825,6 +920,204 @@ struct SparseGemmArguments {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Two dimensional convolution
|
||||
//
|
||||
// OperationKind: Conv2d
|
||||
//
|
||||
struct Conv2dConfiguration {
|
||||
|
||||
conv::SplitKMode split_k_mode;
|
||||
|
||||
/// Conv2d problem size
|
||||
// contains strictly conv2d size (N,H,W,C,K,R,S,P,Q,padding,stride,dilation,mode)
|
||||
// also includes (split_k_slices, groups)
|
||||
conv::Conv2dProblemSize problem_size;
|
||||
|
||||
/// Layout object for activations tensor
|
||||
layout::TensorNHWC layout_activations;
|
||||
|
||||
/// Layout object for filters tensor
|
||||
layout::TensorNHWC layout_filters;
|
||||
|
||||
/// Layout object for source tensor
|
||||
layout::TensorNHWC layout_source;
|
||||
|
||||
/// Layout object for output tensor
|
||||
layout::TensorNHWC layout_output;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Mapping functions (A,B,C -> activation,filter,output)
|
||||
layout::TensorNHWC layout_a(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_activations;
|
||||
case library::ConvKind::kDgrad: return layout_output;
|
||||
case library::ConvKind::kWgrad: return layout_output;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
layout::TensorNHWC layout_b(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_filters;
|
||||
case library::ConvKind::kDgrad: return layout_filters;
|
||||
case library::ConvKind::kWgrad: return layout_activations;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
layout::TensorNHWC layout_c(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_output;
|
||||
case library::ConvKind::kDgrad: return layout_activations;
|
||||
case library::ConvKind::kWgrad: return layout_filters;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/// Three dimensional convolution
|
||||
//
|
||||
// OperationKind: Conv3d
|
||||
//
|
||||
struct Conv3dConfiguration {
|
||||
|
||||
conv::SplitKMode split_k_mode;
|
||||
|
||||
/// Conv2d problem size
|
||||
// contains strictly conv2d size (N,D,H,W,C,K,T,R,S,Z,P,Q,padding,stride,dilation,mode)
|
||||
// also includes (split_k_slices, groups)
|
||||
conv::Conv3dProblemSize problem_size;
|
||||
|
||||
/// Layout object for activations tensor
|
||||
layout::TensorNDHWC layout_activations;
|
||||
|
||||
/// Layout object for filters tensor
|
||||
layout::TensorNDHWC layout_filters;
|
||||
|
||||
/// Layout object for source tensor
|
||||
layout::TensorNDHWC layout_source;
|
||||
|
||||
/// Layout object for output tensor
|
||||
layout::TensorNDHWC layout_output;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Mapping functions (A,B,C -> activation,filter,output)
|
||||
layout::TensorNDHWC layout_a(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_activations;
|
||||
case library::ConvKind::kDgrad: return layout_output;
|
||||
case library::ConvKind::kWgrad: return layout_output;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
layout::TensorNDHWC layout_b(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_filters;
|
||||
case library::ConvKind::kDgrad: return layout_filters;
|
||||
case library::ConvKind::kWgrad: return layout_activations;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
layout::TensorNDHWC layout_c(library::ConvKind const &conv_kind) const {
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return layout_output;
|
||||
case library::ConvKind::kDgrad: return layout_activations;
|
||||
case library::ConvKind::kWgrad: return layout_filters;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Arguments for CONV
|
||||
struct ConvArguments {
|
||||
|
||||
/////////////////////////////////////////////////////////
|
||||
/// ImplicitGemm matrices A, B, C, D
|
||||
/////////////////////////////////////////////////////////
|
||||
/// pointer to implicit gemm matrix A
|
||||
void const *A;
|
||||
|
||||
/// pointer to implicit gemm matrix B
|
||||
void const *B;
|
||||
|
||||
/// pointer to implicit gemm matrix C
|
||||
void const *C;
|
||||
|
||||
/// pointer to implicit gemm desitination matrix D
|
||||
void *D;
|
||||
|
||||
/// Host or device pointer to alpha scalar
|
||||
void const *alpha;
|
||||
|
||||
/// Host or device pointer to beta scalar
|
||||
void const *beta;
|
||||
|
||||
/// Enumerant indicating whether alpha/beta point to host or device memory
|
||||
ScalarPointerMode pointer_mode;
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Configuration for Reduction operations
|
||||
//
|
||||
// OperationKind: Reduction
|
||||
//
|
||||
struct ReductionConfiguration {
|
||||
|
||||
/// Redcution problem size
|
||||
MatrixCoord problem_size;
|
||||
|
||||
/// Number of partitions to reduce
|
||||
int partitions;
|
||||
|
||||
/// Number of lements between each partition
|
||||
int64_t partition_stride;
|
||||
|
||||
/// leading dimension of 'w'orksace operand
|
||||
int64_t ldw;
|
||||
|
||||
/// leading dimension of 's'ource operand
|
||||
int64_t lds;
|
||||
|
||||
/// leading dimension of 'd'estination operand
|
||||
int64_t ldd;
|
||||
};
|
||||
|
||||
/// Arguments for Reduction
|
||||
struct ReductionArguments {
|
||||
|
||||
/// Pointer to workspace matrix
|
||||
void const *workspace;
|
||||
|
||||
/// Pointer to source matrix
|
||||
void const *source;
|
||||
|
||||
/// Pointer to destination matrix
|
||||
void *destination;
|
||||
|
||||
/// pointer to reference matrix
|
||||
void *reference;
|
||||
|
||||
/// Host or device pointer to alpha scalar
|
||||
void const *alpha;
|
||||
|
||||
/// Host or device pointer to beta scalar
|
||||
void const *beta;
|
||||
|
||||
/// Enumerant indicating whether alpha/beta point to host or device memory
|
||||
ScalarPointerMode pointer_mode;
|
||||
};
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
@@ -51,6 +51,9 @@ class Manifest;
|
||||
// init and insert all cutlass gemm operations in manifest object (procedurally generated using generator.py)
|
||||
void initialize_all(Manifest &manifest);
|
||||
|
||||
// init and insert all reduction op in manifest object (manually instantiated in library/reduction)
|
||||
void initialize_all_reduction_op(Manifest &manifest);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// List of operations
|
||||
|
||||
@@ -208,6 +208,262 @@ using GemmOperationFunctionalMap = std::unordered_map<
|
||||
>;
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Data Structures for Conv Functional Maps
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Tuple uniquely identifying conv2d functional behavior
|
||||
struct ConvFunctionalKey {
|
||||
library::Provider provider;
|
||||
library::ConvKind conv_kind;
|
||||
library::NumericTypeID element_A;
|
||||
library::LayoutTypeID layout_A;
|
||||
library::NumericTypeID element_B;
|
||||
library::LayoutTypeID layout_B;
|
||||
library::NumericTypeID element_C;
|
||||
library::LayoutTypeID layout_C;
|
||||
library::NumericTypeID element_accumulator;
|
||||
library::NumericTypeID element_compute;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
inline
|
||||
ConvFunctionalKey(
|
||||
library::Provider provider = library::Provider::kInvalid,
|
||||
library::ConvKind conv_kind = library::ConvKind::kFprop,
|
||||
library::NumericTypeID element_A = library::NumericTypeID::kF16,
|
||||
library::LayoutTypeID layout_A = library::LayoutTypeID::kTensorNHWC,
|
||||
library::NumericTypeID element_B = library::NumericTypeID::kF16,
|
||||
library::LayoutTypeID layout_B = library::LayoutTypeID::kTensorNHWC,
|
||||
library::NumericTypeID element_C = library::NumericTypeID::kF16,
|
||||
library::LayoutTypeID layout_C = library::LayoutTypeID::kTensorNHWC,
|
||||
library::NumericTypeID element_accumulator = library::NumericTypeID::kF32,
|
||||
library::NumericTypeID element_compute = library::NumericTypeID::kF32
|
||||
):
|
||||
provider(provider),
|
||||
conv_kind(conv_kind),
|
||||
element_A(element_A),
|
||||
layout_A(layout_A),
|
||||
element_B(element_B),
|
||||
layout_B(layout_B),
|
||||
element_C(element_C),
|
||||
layout_C(layout_C),
|
||||
element_accumulator(element_accumulator),
|
||||
element_compute(element_compute)
|
||||
{ }
|
||||
|
||||
inline
|
||||
bool operator==(ConvFunctionalKey const &rhs) const {
|
||||
return
|
||||
(provider == rhs.provider) &&
|
||||
(conv_kind == rhs.conv_kind) &&
|
||||
(element_A == rhs.element_A) &&
|
||||
(layout_A == rhs.layout_A) &&
|
||||
(element_B == rhs.element_B) &&
|
||||
(layout_B == rhs.layout_B) &&
|
||||
(element_C == rhs.element_C) &&
|
||||
(layout_C == rhs.layout_C) &&
|
||||
(element_accumulator == rhs.element_accumulator) &&
|
||||
(element_compute == rhs.element_compute);
|
||||
}
|
||||
|
||||
inline
|
||||
bool operator!=(ConvFunctionalKey const &rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
};
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
inline
|
||||
std::ostream& operator<< (std::ostream& out, const cutlass::library::ConvFunctionalKey& key) {
|
||||
out << "{\n"
|
||||
<< "provider: " << to_string(key.provider) << std::endl
|
||||
<< "conv_kind: " << to_string(key.conv_kind) << std::endl
|
||||
<< "element_A: " << to_string(key.element_A) << std::endl
|
||||
<< "layout_A: " << to_string(key.layout_A) << std::endl
|
||||
<< "element_B: " << to_string(key.element_B) << std::endl
|
||||
<< "layout_B: " << to_string(key.layout_B) << std::endl
|
||||
<< "element_C: " << to_string(key.element_C) << std::endl
|
||||
<< "layout_C: " << to_string(key.layout_C) << std::endl
|
||||
<< "element_accumulator: " << to_string(key.element_accumulator) << std::endl
|
||||
<< "element_compute: " << to_string(key.element_compute) << std::endl
|
||||
<< "}";
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
struct ConvFunctionalKeyHasher {
|
||||
using IntHash = std::hash<int>;
|
||||
|
||||
inline
|
||||
static size_t rotl(size_t key, int shl) {
|
||||
return (key << shl) | (key >> (sizeof(key)*8 - shl));
|
||||
}
|
||||
|
||||
inline
|
||||
size_t operator()(ConvFunctionalKey const &key) const {
|
||||
IntHash hash;
|
||||
|
||||
return
|
||||
rotl(hash(int(key.provider)), 1) ^
|
||||
rotl(hash(int(key.conv_kind)), 2) ^
|
||||
rotl(hash(int(key.element_A)), 3) ^
|
||||
rotl(hash(int(key.layout_A)), 4) ^
|
||||
rotl(hash(int(key.element_B)), 5) ^
|
||||
rotl(hash(int(key.layout_B)), 6) ^
|
||||
rotl(hash(int(key.element_C)), 7) ^
|
||||
rotl(hash(int(key.layout_C)), 8) ^
|
||||
rotl(hash(int(key.element_accumulator)), 9) ^
|
||||
rotl(hash(int(key.element_compute)), 10);
|
||||
}
|
||||
};
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Establishes a partial ordering to search for Conv2d operators
|
||||
struct ConvPreferenceKey {
|
||||
|
||||
int compute_capability;
|
||||
IteratorAlgorithmID iterator_algorithm;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
ConvPreferenceKey(): compute_capability(), iterator_algorithm() { }
|
||||
|
||||
ConvPreferenceKey(int cc, IteratorAlgorithmID iterator_algorithm):
|
||||
compute_capability(cc), iterator_algorithm(iterator_algorithm) { }
|
||||
|
||||
bool operator<(ConvPreferenceKey const &rhs) const {
|
||||
return (compute_capability < rhs.compute_capability) ||
|
||||
((compute_capability == rhs.compute_capability) && (iterator_algorithm < rhs.iterator_algorithm));
|
||||
}
|
||||
|
||||
bool operator==(ConvPreferenceKey const &rhs) const {
|
||||
return (compute_capability == rhs.compute_capability) &&
|
||||
(iterator_algorithm == rhs.iterator_algorithm);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Maps minimum compute capability onto a vector of possible operations
|
||||
using ConvOperationVectorMap = std::map<
|
||||
ConvPreferenceKey,
|
||||
std::vector<Operation const *>
|
||||
>;
|
||||
|
||||
/// Maps a GemmFunctionalKey onto a vector of Operation * objects expected to be of kind kGemm
|
||||
using ConvOperationFunctionalMap = std::unordered_map<
|
||||
ConvFunctionalKey,
|
||||
ConvOperationVectorMap,
|
||||
ConvFunctionalKeyHasher
|
||||
>;
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/// Tuple uniquely identifying conv2d functional behavior
|
||||
struct ReductionFunctionalKey {
|
||||
library::Provider provider;
|
||||
library::NumericTypeID element_workspace;
|
||||
library::NumericTypeID element_accumulator;
|
||||
library::NumericTypeID element_output;
|
||||
library::NumericTypeID element_compute;
|
||||
library::MathOperationID reduce_math_op;
|
||||
library::EpilogueKind epilogue_math_op;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
inline
|
||||
ReductionFunctionalKey(
|
||||
library::Provider provider = library::Provider::kInvalid,
|
||||
library::NumericTypeID element_workspace = library::NumericTypeID::kF16,
|
||||
library::NumericTypeID element_accumulator = library::NumericTypeID::kF32,
|
||||
library::NumericTypeID element_output = library::NumericTypeID::kF16,
|
||||
library::NumericTypeID element_compute = library::NumericTypeID::kF32,
|
||||
library::MathOperationID reduce_math_op = library::MathOperationID::kAdd,
|
||||
library::EpilogueKind epilogue_math_op = library::EpilogueKind::kLinearCombination
|
||||
):
|
||||
provider(provider),
|
||||
element_workspace(element_workspace),
|
||||
element_accumulator(element_accumulator),
|
||||
element_output(element_output),
|
||||
element_compute(element_compute),
|
||||
reduce_math_op(reduce_math_op),
|
||||
epilogue_math_op(epilogue_math_op)
|
||||
{ }
|
||||
|
||||
inline
|
||||
bool operator==(ReductionFunctionalKey const &rhs) const {
|
||||
return
|
||||
(provider == rhs.provider) &&
|
||||
(element_workspace == rhs.element_workspace) &&
|
||||
(element_accumulator == rhs.element_accumulator) &&
|
||||
(element_output == rhs.element_output) &&
|
||||
(element_compute == rhs.element_compute) &&
|
||||
(reduce_math_op == rhs.reduce_math_op) &&
|
||||
(epilogue_math_op == rhs.epilogue_math_op);
|
||||
}
|
||||
|
||||
inline
|
||||
bool operator!=(ReductionFunctionalKey const &rhs) const {
|
||||
return !(*this == rhs);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct ReductionFunctionalKeyHasher {
|
||||
using IntHash = std::hash<int>;
|
||||
|
||||
inline
|
||||
static size_t rotl(size_t key, int shl) {
|
||||
return (key << shl) | (key >> (sizeof(key)*8 - shl));
|
||||
}
|
||||
|
||||
inline
|
||||
size_t operator()(ReductionFunctionalKey const &key) const {
|
||||
IntHash hash;
|
||||
|
||||
return
|
||||
rotl(hash(int(key.provider)), 1) ^
|
||||
rotl(hash(int(key.element_workspace)), 2) ^
|
||||
rotl(hash(int(key.element_accumulator)), 3) ^
|
||||
rotl(hash(int(key.element_output)), 4) ^
|
||||
rotl(hash(int(key.element_compute)), 5) ^
|
||||
rotl(hash(int(key.reduce_math_op)), 6) ^
|
||||
rotl(hash(int(key.epilogue_math_op)), 7);
|
||||
}
|
||||
};
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
inline
|
||||
std::ostream& operator<< (std::ostream& out, const ReductionFunctionalKey& key) {
|
||||
out << "{\n"
|
||||
<< "provider: " << library::to_string(key.provider) << std::endl
|
||||
<< "element_workspace : " << library::to_string(key.element_workspace) << std::endl
|
||||
<< "element_accumulator : " << library::to_string(key.element_accumulator) << std::endl
|
||||
<< "element_output : " << library::to_string(key.element_output) << std::endl
|
||||
<< "element_compute : " << library::to_string(key.element_compute) << std::endl
|
||||
<< "}";
|
||||
return out;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// ReductionOperationFunctionalMap has NO preference key and a single instance per functional key
|
||||
// i.e. only one tile size configuration per functional key
|
||||
using ReductionOperationFunctionalMap = std::unordered_map<
|
||||
ReductionFunctionalKey,
|
||||
library::Operation const *,
|
||||
ReductionFunctionalKeyHasher
|
||||
>;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Table of cutlass::library::Operation instances
|
||||
@@ -218,6 +474,18 @@ public:
|
||||
// provider (kCUTLASS)
|
||||
GemmOperationFunctionalMap gemm_operations;
|
||||
|
||||
/// Map of all operations of type kConv2d
|
||||
// provider (kCUTLASS, kReferenceHost, kReferenceDevice)
|
||||
ConvOperationFunctionalMap conv2d_operations;
|
||||
|
||||
/// Map of all operations of type kConv3d
|
||||
// provider (kCUTLASS, kReferenceHost, kReferenceDevice)
|
||||
ConvOperationFunctionalMap conv3d_operations;
|
||||
|
||||
/// Map of all operations of type kConv2d
|
||||
// provider (kCUTLASS)
|
||||
ReductionOperationFunctionalMap reduction_operations;
|
||||
|
||||
public:
|
||||
|
||||
void append(Manifest const &manifest);
|
||||
|
||||
@@ -122,6 +122,27 @@ char const *to_string(SplitKMode split_k_mode, bool pretty = false);
|
||||
template <>
|
||||
SplitKMode from_string<SplitKMode>(std::string const &str);
|
||||
|
||||
/// Converts a ConvModeID enumerant to a string
|
||||
char const *to_string(ConvModeID type, bool pretty = false);
|
||||
|
||||
/// Converts a ConvModeID enumerant from a string
|
||||
template <>
|
||||
ConvModeID from_string<ConvModeID>(std::string const &str);
|
||||
|
||||
/// Converts a IteratorAlgorithmID enumerant to a string
|
||||
char const *to_string(IteratorAlgorithmID type, bool pretty = false);
|
||||
|
||||
/// Converts a IteratorAlgorithmID enumerant from a string
|
||||
template <>
|
||||
IteratorAlgorithmID from_string<IteratorAlgorithmID>(std::string const &str);
|
||||
|
||||
/// Converts a ConvKind enumerant to a string
|
||||
char const *to_string(ConvKind type, bool pretty = false);
|
||||
|
||||
/// Converts a ConvKind enumerant from a string
|
||||
template <>
|
||||
ConvKind from_string<ConvKind>(std::string const &str);
|
||||
|
||||
/// Lexical cast from int64_t to string
|
||||
std::string lexical_cast(int64_t int_value);
|
||||
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
#
|
||||
# \file generator.py
|
||||
#
|
||||
# \brief Generates the CUTLASS Library's instances
|
||||
#
|
||||
#
|
||||
|
||||
import enum
|
||||
import os.path
|
||||
import shutil
|
||||
|
||||
from library import *
|
||||
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
class Conv2dOperation:
|
||||
#
|
||||
def __init__(self, conv_kind, iterator_algorithm, arch, tile_description, A, B, C, element_epilogue, \
|
||||
stride_support, epilogue_functor = EpilogueFunctor.LinearCombination, swizzling_functor = SwizzlingFunctor.Identity4):
|
||||
|
||||
self.operation_kind = OperationKind.Conv2d
|
||||
self.arch = arch
|
||||
self.tile_description = tile_description
|
||||
self.conv_kind = conv_kind
|
||||
self.A = A
|
||||
self.B = B
|
||||
self.C = C
|
||||
self.element_epilogue = element_epilogue
|
||||
self.epilogue_functor = epilogue_functor
|
||||
self.iterator_algorithm = iterator_algorithm
|
||||
self.stride_support = stride_support
|
||||
self.swizzling_functor = swizzling_functor
|
||||
#
|
||||
def is_complex(self):
|
||||
complex_operators = [
|
||||
MathOperation.multiply_add_complex,
|
||||
MathOperation.multiply_add_complex_gaussian
|
||||
]
|
||||
return self.tile_description.math_instruction.math_operation in complex_operators
|
||||
|
||||
#
|
||||
def accumulator_type(self):
|
||||
accum = self.tile_description.math_instruction.element_accumulator
|
||||
|
||||
if self.is_complex():
|
||||
return get_complex_from_real(accum)
|
||||
|
||||
return accum
|
||||
|
||||
#
|
||||
def core_name(self):
|
||||
''' The basic operation kind is prefixed with a letter indicating the accumulation type. '''
|
||||
|
||||
intermediate_type = ''
|
||||
|
||||
if self.tile_description.math_instruction.opcode_class == OpcodeClass.TensorOp:
|
||||
inst_shape = "%d%d%d" % tuple(self.tile_description.math_instruction.instruction_shape)
|
||||
if self.tile_description.math_instruction.element_a != self.A.element and \
|
||||
self.tile_description.math_instruction.element_a != self.accumulator_type():
|
||||
intermediate_type = DataTypeNames[self.tile_description.math_instruction.element_a]
|
||||
else:
|
||||
inst_shape = ''
|
||||
|
||||
return "%s%s%s%s_%s" % (ShortDataTypeNames[self.accumulator_type()], \
|
||||
inst_shape, intermediate_type, ConvKindNames[self.conv_kind], IteratorAlgorithmNames[self.iterator_algorithm])
|
||||
|
||||
#
|
||||
def extended_name(self):
|
||||
''' Append data types if they differ from compute type. '''
|
||||
if self.C.element != self.tile_description.math_instruction.element_accumulator and \
|
||||
self.A.element != self.tile_description.math_instruction.element_accumulator:
|
||||
extended_name = "${element_c}_${core_name}_${element_a}"
|
||||
elif self.C.element == self.tile_description.math_instruction.element_accumulator and \
|
||||
self.A.element != self.tile_description.math_instruction.element_accumulator:
|
||||
extended_name = "${core_name}_${element_a}"
|
||||
else:
|
||||
extended_name = "${core_name}"
|
||||
|
||||
extended_name = SubstituteTemplate(extended_name, {
|
||||
'element_a': DataTypeNames[self.A.element],
|
||||
'element_c': DataTypeNames[self.C.element],
|
||||
'core_name': self.core_name()
|
||||
})
|
||||
|
||||
return extended_name
|
||||
|
||||
#
|
||||
def layout_name(self):
|
||||
return "%s" % (ShortLayoutTypeNames[self.A.layout])
|
||||
|
||||
#
|
||||
def configuration_name(self):
|
||||
''' The full procedural name indicates architecture, extended name, tile size, and layout. '''
|
||||
|
||||
opcode_class_name = OpcodeClassNames[self.tile_description.math_instruction.opcode_class]
|
||||
|
||||
threadblock = "%dx%d_%dx%d" % (
|
||||
self.tile_description.threadblock_shape[0],
|
||||
self.tile_description.threadblock_shape[1],
|
||||
self.tile_description.threadblock_shape[2],
|
||||
self.tile_description.stages
|
||||
)
|
||||
|
||||
if self.stride_support == StrideSupport.Unity:
|
||||
configuration_name = "cutlass_${opcode_class}_${extended_name}_${threadblock}_${layout}_unity_stride"
|
||||
else:
|
||||
configuration_name = "cutlass_${opcode_class}_${extended_name}_${threadblock}_${layout}"
|
||||
|
||||
return SubstituteTemplate(
|
||||
configuration_name,
|
||||
{
|
||||
'opcode_class': opcode_class_name,
|
||||
'extended_name': self.extended_name(),
|
||||
'threadblock': threadblock,
|
||||
'layout': self.layout_name(),
|
||||
}
|
||||
)
|
||||
|
||||
#
|
||||
def procedural_name(self):
|
||||
''' The full procedural name indicates architecture, extended name, tile size, and layout. '''
|
||||
return self.configuration_name()
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Emits single instances of a CUTLASS device-wide operator
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
class EmitConv2dInstance:
|
||||
def __init__(self):
|
||||
self.template = """
|
||||
// Conv2d${conv_kind_name} ${iterator_algorithm_name} kernel instance "${operation_name}"
|
||||
using ${operation_name}_base =
|
||||
typename cutlass::conv::kernel::DefaultConv2d${conv_kind_name}<
|
||||
${element_a},
|
||||
${layout_a},
|
||||
${element_b},
|
||||
${layout_b},
|
||||
${element_c},
|
||||
${layout_c},
|
||||
${element_accumulator},
|
||||
${opcode_class},
|
||||
${arch},
|
||||
cutlass::gemm::GemmShape<${threadblock_shape_m}, ${threadblock_shape_n}, ${threadblock_shape_k}>,
|
||||
cutlass::gemm::GemmShape<${warp_shape_m}, ${warp_shape_n}, ${warp_shape_k} >,
|
||||
cutlass::gemm::GemmShape<${instruction_shape_m}, ${instruction_shape_n}, ${instruction_shape_k}>,
|
||||
${epilogue_functor}<
|
||||
${element_c},
|
||||
${epilogue_vector_length},
|
||||
${element_accumulator},
|
||||
${element_epilogue}
|
||||
>,
|
||||
${swizzling_functor}, // cutlass::gemm::threadblock::GemmSplitKIdentityThreadblockSwizzle<>,
|
||||
${stages},
|
||||
${math_operator},
|
||||
${iterator_algorithm},
|
||||
${stride_support}
|
||||
>::Kernel;
|
||||
"""
|
||||
|
||||
|
||||
def emit(self, operation):
|
||||
|
||||
warp_shape = [int(operation.tile_description.threadblock_shape[idx] / operation.tile_description.warp_count[idx]) for idx in range(3)]
|
||||
|
||||
epilogue_vector_length = int(min(operation.C.alignment * DataTypeSize[operation.C.element], 128) / DataTypeSize[operation.C.element])
|
||||
|
||||
values = {
|
||||
'operation_name': operation.procedural_name(),
|
||||
'conv_kind': ConvKindTag[operation.conv_kind],
|
||||
'conv_kind_name': ConvKindNames[operation.conv_kind].capitalize(),
|
||||
'element_a': DataTypeTag[operation.A.element],
|
||||
'layout_a': LayoutTag[operation.A.layout],
|
||||
'element_b': DataTypeTag[operation.B.element],
|
||||
'layout_b': LayoutTag[operation.B.layout],
|
||||
'element_c': DataTypeTag[operation.C.element],
|
||||
'layout_c': LayoutTag[operation.C.layout],
|
||||
'element_accumulator': DataTypeTag[operation.accumulator_type()],
|
||||
'opcode_class': OpcodeClassTag[operation.tile_description.math_instruction.opcode_class],
|
||||
'arch': "cutlass::arch::Sm%d" % operation.arch,
|
||||
'threadblock_shape_m': str(operation.tile_description.threadblock_shape[0]),
|
||||
'threadblock_shape_n': str(operation.tile_description.threadblock_shape[1]),
|
||||
'threadblock_shape_k': str(operation.tile_description.threadblock_shape[2]),
|
||||
'warp_shape_m': str(warp_shape[0]),
|
||||
'warp_shape_n': str(warp_shape[1]),
|
||||
'warp_shape_k': str(warp_shape[2]),
|
||||
'instruction_shape_m': str(operation.tile_description.math_instruction.instruction_shape[0]),
|
||||
'instruction_shape_n': str(operation.tile_description.math_instruction.instruction_shape[1]),
|
||||
'instruction_shape_k': str(operation.tile_description.math_instruction.instruction_shape[2]),
|
||||
'epilogue_vector_length': str(epilogue_vector_length),
|
||||
'epilogue_functor': EpilogueFunctorTag[operation.epilogue_functor],
|
||||
'element_epilogue': str(DataTypeTag[operation.element_epilogue]),
|
||||
'swizzling_functor': SwizzlingFunctorTag[operation.swizzling_functor],
|
||||
'stages': str(operation.tile_description.stages),
|
||||
'iterator_algorithm': IteratorAlgorithmTag[operation.iterator_algorithm],
|
||||
'iterator_algorithm_name': IteratorAlgorithmNames[operation.iterator_algorithm].capitalize(),
|
||||
'stride_support': StrideSupportTag[operation.stride_support],
|
||||
'math_operator': 'cutlass::arch::OpMultiplyAddComplex' if operation.is_complex() else \
|
||||
MathOperationTag[operation.tile_description.math_instruction.math_operation]
|
||||
}
|
||||
|
||||
return SubstituteTemplate(self.template, values)
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Generator functions for all layouts
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
def GenerateConv2dTensorOp(manifest, tile_descriptions, min_cc, align = 128):
|
||||
|
||||
for tile in tile_descriptions:
|
||||
for conv_kind in [ConvKind.Fprop, ConvKind.Dgrad, ConvKind.Wgrad]:
|
||||
|
||||
if conv_kind == ConvKind.Fprop or (tile.math_instruction.element_accumulator in [DataType.f16, DataType.f32]):
|
||||
|
||||
#
|
||||
output_types = [tile.math_instruction.element_a, tile.math_instruction.element_accumulator] \
|
||||
if DataTypeSize[tile.math_instruction.element_accumulator] == 32 \
|
||||
else [tile.math_instruction.element_accumulator,]
|
||||
|
||||
for output_type in output_types:
|
||||
A = TensorDescription(tile.math_instruction.element_a, LayoutType.TensorNHWC, int(align / DataTypeSize[tile.math_instruction.element_a]))
|
||||
B = TensorDescription(tile.math_instruction.element_b, LayoutType.TensorNHWC, int(align / DataTypeSize[tile.math_instruction.element_b]))
|
||||
C = TensorDescription(output_type, LayoutType.TensorNHWC, max(1, int(align / DataTypeSize[output_type])))
|
||||
|
||||
manifest.append(Conv2dOperation(conv_kind, min_cc, tile, A, B, C, tile.math_instruction.element_accumulator))
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Emitters functions for all targets
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
class EmitConv2dConfigurationLibrary:
|
||||
def __init__(self, operation_path, configuration_name):
|
||||
self.configuration_name = configuration_name
|
||||
self.configuration_path = os.path.join(operation_path, "%s.cu" % configuration_name)
|
||||
|
||||
self.instance_emitter = EmitConv2dInstance()
|
||||
|
||||
self.instance_template = """
|
||||
${operation_instance}
|
||||
|
||||
// Derived class
|
||||
struct ${operation_name} :
|
||||
public ${operation_name}_base { };
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
"""
|
||||
self.header_template = """
|
||||
/*
|
||||
Generated by conv2d_operation.py - Do not edit.
|
||||
*/
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "library_internal.h"
|
||||
#include "conv2d_operation.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
"""
|
||||
|
||||
self.configuration_header = """
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
// Initialize all instances
|
||||
void initialize_${configuration_name}(Manifest &manifest) {
|
||||
|
||||
"""
|
||||
|
||||
self.configuration_instance = """
|
||||
using Operation_${operation_name} = cutlass::conv::device::ImplicitGemmConvolution<
|
||||
${operation_name}>;
|
||||
|
||||
manifest.append(new cutlass::library::Conv2dOperation<
|
||||
Operation_${operation_name}>(
|
||||
"${operation_name}"));
|
||||
|
||||
"""
|
||||
|
||||
self.configuration_epilogue = """
|
||||
}
|
||||
"""
|
||||
self.epilogue_template = """
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
"""
|
||||
|
||||
#
|
||||
def __enter__(self):
|
||||
self.configuration_file = open(self.configuration_path, "w")
|
||||
self.configuration_file.write(SubstituteTemplate(self.header_template, {
|
||||
'configuration_name': self.configuration_name
|
||||
}))
|
||||
self.operations = []
|
||||
return self
|
||||
|
||||
#
|
||||
def emit(self, operation):
|
||||
self.operations.append(operation)
|
||||
self.configuration_file.write(SubstituteTemplate(self.instance_template, {
|
||||
'configuration_name': self.configuration_name,
|
||||
'operation_name': operation.procedural_name(),
|
||||
'operation_instance': self.instance_emitter.emit(operation)
|
||||
}))
|
||||
|
||||
#
|
||||
def __exit__(self, exception_type, exception_value, traceback):
|
||||
|
||||
self.configuration_file.write(SubstituteTemplate(self.configuration_header, {
|
||||
'configuration_name': self.configuration_name
|
||||
}))
|
||||
|
||||
for operation in self.operations:
|
||||
self.configuration_file.write(SubstituteTemplate(self.configuration_instance, {
|
||||
'configuration_name': self.configuration_name,
|
||||
'operation_name': operation.procedural_name()
|
||||
}))
|
||||
|
||||
self.configuration_file.write(self.configuration_epilogue)
|
||||
self.configuration_file.write(self.epilogue_template)
|
||||
self.configuration_file.close()
|
||||
|
||||
|
||||
###################################################################################################
|
||||
###################################################################################################
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
#
|
||||
# \file generator.py
|
||||
#
|
||||
# \brief Generates the CUTLASS Library's instances
|
||||
#
|
||||
#
|
||||
|
||||
import enum
|
||||
import os.path
|
||||
import shutil
|
||||
|
||||
from library import *
|
||||
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
class Conv3dOperation:
|
||||
#
|
||||
def __init__(self, conv_kind, iterator_algorithm, arch, tile_description, A, B, C, element_epilogue, \
|
||||
stride_support, epilogue_functor = EpilogueFunctor.LinearCombination, swizzling_functor = SwizzlingFunctor.Identity4):
|
||||
|
||||
self.operation_kind = OperationKind.Conv3d
|
||||
self.arch = arch
|
||||
self.tile_description = tile_description
|
||||
self.conv_kind = conv_kind
|
||||
self.A = A
|
||||
self.B = B
|
||||
self.C = C
|
||||
self.element_epilogue = element_epilogue
|
||||
self.epilogue_functor = epilogue_functor
|
||||
self.iterator_algorithm = iterator_algorithm
|
||||
self.stride_support = stride_support
|
||||
self.swizzling_functor = swizzling_functor
|
||||
|
||||
#
|
||||
def core_name(self):
|
||||
''' The basic operation kind is prefixed with a letter indicating the accumulation type. '''
|
||||
|
||||
intermediate_type = ''
|
||||
|
||||
if self.tile_description.math_instruction.opcode_class == OpcodeClass.TensorOp:
|
||||
inst_shape = "%d%d%d" % tuple(self.tile_description.math_instruction.instruction_shape)
|
||||
if self.tile_description.math_instruction.element_a != self.A.element and \
|
||||
self.tile_description.math_instruction.element_a != self.tile_description.math_instruction.element_accumulator:
|
||||
intermediate_type = DataTypeNames[self.tile_description.math_instruction.element_a]
|
||||
else:
|
||||
inst_shape = ''
|
||||
|
||||
return "%s%s%s%s3d_%s" % (ShortDataTypeNames[self.tile_description.math_instruction.element_accumulator], \
|
||||
inst_shape, intermediate_type, ConvKindNames[self.conv_kind], IteratorAlgorithmNames[self.iterator_algorithm])
|
||||
|
||||
#
|
||||
def extended_name(self):
|
||||
''' Append data types if they differ from compute type. '''
|
||||
if self.C.element != self.tile_description.math_instruction.element_accumulator and \
|
||||
self.A.element != self.tile_description.math_instruction.element_accumulator:
|
||||
extended_name = "${element_c}_${core_name}_${element_a}"
|
||||
elif self.C.element == self.tile_description.math_instruction.element_accumulator and \
|
||||
self.A.element != self.tile_description.math_instruction.element_accumulator:
|
||||
extended_name = "${core_name}_${element_a}"
|
||||
else:
|
||||
extended_name = "${core_name}"
|
||||
|
||||
extended_name = SubstituteTemplate(extended_name, {
|
||||
'element_a': DataTypeNames[self.A.element],
|
||||
'element_c': DataTypeNames[self.C.element],
|
||||
'core_name': self.core_name()
|
||||
})
|
||||
|
||||
return extended_name
|
||||
|
||||
#
|
||||
def configuration_name(self):
|
||||
''' The full procedural name indicates architecture, extended name, tile size, and layout. '''
|
||||
|
||||
opcode_class_name = OpcodeClassNames[self.tile_description.math_instruction.opcode_class]
|
||||
|
||||
threadblock = "%dx%d_%dx%d" % (
|
||||
self.tile_description.threadblock_shape[0],
|
||||
self.tile_description.threadblock_shape[1],
|
||||
self.tile_description.threadblock_shape[2],
|
||||
self.tile_description.stages
|
||||
)
|
||||
|
||||
if self.stride_support == StrideSupport.Unity:
|
||||
configuration_name = "cutlass_${opcode_class}_${extended_name}_${threadblock}_unity_stride"
|
||||
else:
|
||||
configuration_name = "cutlass_${opcode_class}_${extended_name}_${threadblock}"
|
||||
|
||||
return SubstituteTemplate(
|
||||
configuration_name,
|
||||
{
|
||||
'opcode_class': opcode_class_name,
|
||||
'extended_name': self.extended_name(),
|
||||
'threadblock': threadblock,
|
||||
}
|
||||
)
|
||||
|
||||
#
|
||||
def procedural_name(self):
|
||||
''' The full procedural name indicates architecture, extended name, tile size, and layout. '''
|
||||
return self.configuration_name()
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Emits single instances of a CUTLASS device-wide operator
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
class EmitConv3dInstance:
|
||||
def __init__(self):
|
||||
self.template = """
|
||||
// Conv3d${conv_kind_name} ${iterator_algorithm_name} kernel instance "${operation_name}"
|
||||
using ${operation_name}_base =
|
||||
typename cutlass::conv::kernel::DefaultConv3d${conv_kind_name}<
|
||||
${element_a},
|
||||
cutlass::layout::TensorNDHWC,
|
||||
${element_b},
|
||||
cutlass::layout::TensorNDHWC,
|
||||
${element_c},
|
||||
cutlass::layout::TensorNDHWC,
|
||||
${element_accumulator},
|
||||
${opcode_class},
|
||||
${arch},
|
||||
cutlass::gemm::GemmShape<${threadblock_shape_m}, ${threadblock_shape_n}, ${threadblock_shape_k}>,
|
||||
cutlass::gemm::GemmShape<${warp_shape_m}, ${warp_shape_n}, ${warp_shape_k} >,
|
||||
cutlass::gemm::GemmShape<${instruction_shape_m}, ${instruction_shape_n}, ${instruction_shape_k}>,
|
||||
${epilogue_functor}<
|
||||
${element_c},
|
||||
${epilogue_vector_length},
|
||||
${element_accumulator},
|
||||
${element_epilogue}
|
||||
>,
|
||||
${swizzling_functor}, // cutlass::gemm::threadblock::GemmSplitKIdentityThreadblockSwizzle<>,
|
||||
${stages},
|
||||
cutlass::arch::OpMultiplyAdd,
|
||||
${iterator_algorithm},
|
||||
${stride_support}
|
||||
>::Kernel;
|
||||
"""
|
||||
|
||||
|
||||
def emit(self, operation):
|
||||
|
||||
warp_shape = [int(operation.tile_description.threadblock_shape[idx] / operation.tile_description.warp_count[idx]) for idx in range(3)]
|
||||
|
||||
epilogue_vector_length = int(min(operation.C.alignment * DataTypeSize[operation.C.element], 128) / DataTypeSize[operation.C.element])
|
||||
|
||||
values = {
|
||||
'operation_name': operation.procedural_name(),
|
||||
'conv_kind': ConvKindTag[operation.conv_kind],
|
||||
'conv_kind_name': ConvKindNames[operation.conv_kind].capitalize(),
|
||||
'element_a': DataTypeTag[operation.A.element],
|
||||
'layout_a': LayoutTag[operation.A.layout],
|
||||
'element_b': DataTypeTag[operation.B.element],
|
||||
'layout_b': LayoutTag[operation.B.layout],
|
||||
'element_c': DataTypeTag[operation.C.element],
|
||||
'layout_c': LayoutTag[operation.C.layout],
|
||||
'element_accumulator': DataTypeTag[operation.tile_description.math_instruction.element_accumulator],
|
||||
'opcode_class': OpcodeClassTag[operation.tile_description.math_instruction.opcode_class],
|
||||
'arch': "cutlass::arch::Sm%d" % operation.arch,
|
||||
'threadblock_shape_m': str(operation.tile_description.threadblock_shape[0]),
|
||||
'threadblock_shape_n': str(operation.tile_description.threadblock_shape[1]),
|
||||
'threadblock_shape_k': str(operation.tile_description.threadblock_shape[2]),
|
||||
'warp_shape_m': str(warp_shape[0]),
|
||||
'warp_shape_n': str(warp_shape[1]),
|
||||
'warp_shape_k': str(warp_shape[2]),
|
||||
'instruction_shape_m': str(operation.tile_description.math_instruction.instruction_shape[0]),
|
||||
'instruction_shape_n': str(operation.tile_description.math_instruction.instruction_shape[1]),
|
||||
'instruction_shape_k': str(operation.tile_description.math_instruction.instruction_shape[2]),
|
||||
'epilogue_vector_length': str(epilogue_vector_length),
|
||||
'epilogue_functor': EpilogueFunctorTag[operation.epilogue_functor],
|
||||
'element_epilogue': str(DataTypeTag[operation.element_epilogue]),
|
||||
'swizzling_functor': SwizzlingFunctorTag[operation.swizzling_functor],
|
||||
'stages': str(operation.tile_description.stages),
|
||||
'iterator_algorithm': IteratorAlgorithmTag[operation.iterator_algorithm],
|
||||
'iterator_algorithm_name': IteratorAlgorithmNames[operation.iterator_algorithm].capitalize(),
|
||||
'stride_support': StrideSupportTag[operation.stride_support]
|
||||
}
|
||||
|
||||
return SubstituteTemplate(self.template, values)
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Generator functions for all layouts
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
def GenerateConv3dTensorOp(manifest, tile_descriptions, min_cc, align = 128):
|
||||
|
||||
for tile in tile_descriptions:
|
||||
for conv_kind in [ConvKind.Fprop, ConvKind.Dgrad, ConvKind.Wgrad]:
|
||||
|
||||
if conv_kind == ConvKind.Fprop or (tile.math_instruction.element_accumulator in [DataType.f16, DataType.f32]):
|
||||
|
||||
#
|
||||
output_types = [tile.math_instruction.element_a, tile.math_instruction.element_accumulator] \
|
||||
if DataTypeSize[tile.math_instruction.element_accumulator] == 32 \
|
||||
else [tile.math_instruction.element_accumulator,]
|
||||
|
||||
for output_type in output_types:
|
||||
A = TensorDescription(tile.math_instruction.element_a, LayoutType.TensorNDHWC, int(align / DataTypeSize[tile.math_instruction.element_a]))
|
||||
B = TensorDescription(tile.math_instruction.element_b, LayoutType.TensorNDHWC, int(align / DataTypeSize[tile.math_instruction.element_b]))
|
||||
C = TensorDescription(output_type, LayoutType.TensorNDHWC, max(1, int(align / DataTypeSize[output_type])))
|
||||
|
||||
manifest.append(Conv3dOperation(conv_kind, min_cc, tile, A, B, C, tile.math_instruction.element_accumulator))
|
||||
|
||||
###################################################################################################
|
||||
#
|
||||
# Emitters functions for all targets
|
||||
#
|
||||
###################################################################################################
|
||||
|
||||
class EmitConv3dConfigurationLibrary:
|
||||
def __init__(self, operation_path, configuration_name):
|
||||
self.configuration_name = configuration_name
|
||||
self.configuration_path = os.path.join(operation_path, "%s.cu" % configuration_name)
|
||||
|
||||
self.instance_emitter = EmitConv3dInstance()
|
||||
|
||||
self.instance_template = """
|
||||
${operation_instance}
|
||||
|
||||
// Derived class
|
||||
struct ${operation_name} :
|
||||
public ${operation_name}_base { };
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
"""
|
||||
self.header_template = """
|
||||
/*
|
||||
Generated by conv3d_operation.py - Do not edit.
|
||||
*/
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "library_internal.h"
|
||||
#include "conv3d_operation.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
"""
|
||||
|
||||
self.configuration_header = """
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
// Initialize all instances
|
||||
void initialize_${configuration_name}(Manifest &manifest) {
|
||||
|
||||
"""
|
||||
|
||||
self.configuration_instance = """
|
||||
using Operation_${operation_name} = cutlass::conv::device::ImplicitGemmConvolution<
|
||||
${operation_name}>;
|
||||
|
||||
manifest.append(new cutlass::library::Conv3dOperation<
|
||||
Operation_${operation_name}>(
|
||||
"${operation_name}"));
|
||||
|
||||
"""
|
||||
|
||||
self.configuration_epilogue = """
|
||||
}
|
||||
"""
|
||||
self.epilogue_template = """
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
"""
|
||||
|
||||
#
|
||||
def __enter__(self):
|
||||
self.configuration_file = open(self.configuration_path, "w")
|
||||
self.configuration_file.write(SubstituteTemplate(self.header_template, {
|
||||
'configuration_name': self.configuration_name
|
||||
}))
|
||||
self.operations = []
|
||||
return self
|
||||
|
||||
#
|
||||
def emit(self, operation):
|
||||
self.operations.append(operation)
|
||||
self.configuration_file.write(SubstituteTemplate(self.instance_template, {
|
||||
'configuration_name': self.configuration_name,
|
||||
'operation_name': operation.procedural_name(),
|
||||
'operation_instance': self.instance_emitter.emit(operation)
|
||||
}))
|
||||
|
||||
#
|
||||
def __exit__(self, exception_type, exception_value, traceback):
|
||||
|
||||
self.configuration_file.write(SubstituteTemplate(self.configuration_header, {
|
||||
'configuration_name': self.configuration_name
|
||||
}))
|
||||
|
||||
for operation in self.operations:
|
||||
self.configuration_file.write(SubstituteTemplate(self.configuration_instance, {
|
||||
'configuration_name': self.configuration_name,
|
||||
'operation_name': operation.procedural_name()
|
||||
}))
|
||||
|
||||
self.configuration_file.write(self.configuration_epilogue)
|
||||
self.configuration_file.write(self.epilogue_template)
|
||||
self.configuration_file.close()
|
||||
|
||||
|
||||
###################################################################################################
|
||||
###################################################################################################
|
||||
|
||||
+360
-126
@@ -11,7 +11,6 @@ import argparse
|
||||
|
||||
from library import *
|
||||
from manifest import *
|
||||
from gemm_operation import *
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
@@ -118,10 +117,9 @@ def CreateGemmPlanarComplexOperator(manifest, layouts, tile_descriptions, data_t
|
||||
|
||||
gemm_kinds = [GemmKind.PlanarComplex, GemmKind.PlanarComplexArray]
|
||||
|
||||
# by default, only generate the largest tile and largest alignment
|
||||
# by default, planar complex gemm kernels are not generated
|
||||
if manifest.args.kernels == '':
|
||||
tile_descriptions = [tile_descriptions[0],]
|
||||
alignment_constraints = [alignment_constraints[0],]
|
||||
return
|
||||
|
||||
for gemm_kind in gemm_kinds:
|
||||
for layout in layouts:
|
||||
@@ -141,6 +139,103 @@ def CreateGemmPlanarComplexOperator(manifest, layouts, tile_descriptions, data_t
|
||||
return
|
||||
|
||||
###########################################################################################################
|
||||
# ConvolutionOperator support variations
|
||||
# ____________________________________________________________________
|
||||
# ConvolutionalOperator | Analytic | Optimized
|
||||
# ____________________________________________________________________
|
||||
# | Fprop | (strided) | (strided)
|
||||
# | Dgrad | (strided, unity*) | (unity)
|
||||
# | Wgrad | (strided) | (strided)
|
||||
# ____________________________________________________________________
|
||||
#
|
||||
# Note : Operator marked (*) are supported but not generated to keep the instantiated kernel count low
|
||||
###########################################################################################################
|
||||
# Convolution for 2D operations
|
||||
def CreateConv2dOperator(manifest, layout, tile_descriptions, data_type, alignment, \
|
||||
conv_kinds = [ConvKind.Fprop, ConvKind.Dgrad, ConvKind.Wgrad], epilogue_functor = EpilogueFunctor.LinearCombination):
|
||||
|
||||
element_a, element_b, element_c, element_epilogue = data_type
|
||||
|
||||
# one exceptional case
|
||||
alignment_c = min(8, alignment)
|
||||
|
||||
# iterator algorithm (analytic and optimized)
|
||||
iterator_algorithms = [IteratorAlgorithm.Analytic, IteratorAlgorithm.Optimized]
|
||||
|
||||
# by default, only generate the largest tile size
|
||||
if manifest.args.kernels == '':
|
||||
tile_descriptions = [tile_descriptions[0],]
|
||||
|
||||
operations = []
|
||||
|
||||
for tile in tile_descriptions:
|
||||
for conv_kind in conv_kinds:
|
||||
for iterator_algorithm in iterator_algorithms:
|
||||
A = TensorDescription(element_a, layout[0], alignment)
|
||||
B = TensorDescription(element_b, layout[1], alignment)
|
||||
C = TensorDescription(element_c, layout[2], alignment_c)
|
||||
|
||||
# unity stride only for Optimized Dgrad
|
||||
if (iterator_algorithm == IteratorAlgorithm.Optimized) and (conv_kind == ConvKind.Dgrad):
|
||||
new_operation = Conv2dOperation(conv_kind, iterator_algorithm, tile.minimum_compute_capability, tile,\
|
||||
A, B, C, element_epilogue, StrideSupport.Unity, epilogue_functor)
|
||||
|
||||
manifest.append(new_operation)
|
||||
operations.append(new_operation)
|
||||
|
||||
# strided dgrad is not supported by Optimized Dgrad
|
||||
if (iterator_algorithm == IteratorAlgorithm.Optimized) and (conv_kind == ConvKind.Dgrad):
|
||||
continue
|
||||
|
||||
# strided support for Fprop (Analytic/Optimized), Dgrad (Analytic), and Wgrad (Analytic)
|
||||
new_operation = Conv2dOperation(conv_kind, iterator_algorithm, tile.minimum_compute_capability, tile,\
|
||||
A, B, C, element_epilogue, StrideSupport.Strided, epilogue_functor)
|
||||
|
||||
manifest.append(new_operation)
|
||||
operations.append(new_operation)
|
||||
|
||||
return operations
|
||||
|
||||
# Convolution for 3D operations
|
||||
def CreateConv3dOperator(manifest, layout, tile_descriptions, data_type, alignment, \
|
||||
conv_kinds = [ConvKind.Fprop, ConvKind.Dgrad, ConvKind.Wgrad], epilogue_functor = EpilogueFunctor.LinearCombination):
|
||||
|
||||
element_a, element_b, element_c, element_epilogue = data_type
|
||||
|
||||
# one exceptional case
|
||||
alignment_c = min(8, alignment)
|
||||
|
||||
# iterator algorithm (analytic and optimized)
|
||||
iterator_algorithms = [IteratorAlgorithm.Analytic, IteratorAlgorithm.Optimized]
|
||||
|
||||
# by default, only generate the largest tile size
|
||||
if manifest.args.kernels == '':
|
||||
tile_descriptions = [tile_descriptions[0],]
|
||||
|
||||
operations = []
|
||||
|
||||
for tile in tile_descriptions:
|
||||
for conv_kind in conv_kinds:
|
||||
for iterator_algorithm in iterator_algorithms:
|
||||
A = TensorDescription(element_a, layout, alignment)
|
||||
B = TensorDescription(element_b, layout, alignment)
|
||||
C = TensorDescription(element_c, layout, alignment_c)
|
||||
|
||||
# optimized conv3d iterator algorithm is only for Wgrad
|
||||
if (iterator_algorithm == IteratorAlgorithm.Optimized) \
|
||||
and ((conv_kind == ConvKind.Fprop) or (conv_kind == ConvKind.Dgrad)):
|
||||
continue
|
||||
|
||||
# strided support for Fprop (Analytic/Optimized), Dgrad (Analytic), and Wgrad (Analytic)
|
||||
new_operation = Conv3dOperation(conv_kind, iterator_algorithm, tile.minimum_compute_capability, tile,\
|
||||
A, B, C, element_epilogue, StrideSupport.Strided, epilogue_functor)
|
||||
|
||||
manifest.append(new_operation)
|
||||
operations.append(new_operation)
|
||||
|
||||
|
||||
return operations
|
||||
|
||||
###################################################################################################
|
||||
###################################################################################################
|
||||
|
||||
@@ -191,11 +286,57 @@ def GenerateSM50_Simt(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
if math_inst.element_a == DataType.f32:
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 1)
|
||||
#
|
||||
|
||||
#
|
||||
def GenerateSM50_Simt_complex(manifest, args):
|
||||
layouts = [
|
||||
(LayoutType.ColumnMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.ColumnMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
]
|
||||
|
||||
math_instructions = [
|
||||
MathInstruction( \
|
||||
[1, 1, 1], \
|
||||
DataType.f32, DataType.f32, DataType.f32, \
|
||||
OpcodeClass.Simt, \
|
||||
MathOperation.multiply_add_complex),
|
||||
]
|
||||
|
||||
min_cc = 50
|
||||
max_cc = 1024
|
||||
|
||||
alignment_constraints = [1,]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
tile_descriptions = [
|
||||
TileDescription([128, 128, 8], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
|
||||
data_type = [
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
]
|
||||
|
||||
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 1)
|
||||
#
|
||||
|
||||
#
|
||||
def GenerateSM50(manifest, args):
|
||||
GenerateSM50_Simt(manifest, args)
|
||||
GenerateSM50_Simt_complex(manifest, args)
|
||||
|
||||
###################################################################################################
|
||||
###################################################################################################
|
||||
@@ -362,6 +503,9 @@ def GenerateSM70_TensorOp_884(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 8)
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
if math_inst.element_a != math_inst.element_accumulator:
|
||||
|
||||
@@ -375,6 +519,8 @@ def GenerateSM70_TensorOp_884(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints)
|
||||
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type_mixed, 8)
|
||||
|
||||
#
|
||||
def GenerateSM70_PlanarComplexTensorOp_884(manifest, args):
|
||||
|
||||
@@ -504,50 +650,10 @@ def GenerateSM70_WmmaTensorOp_161616(manifest, args):
|
||||
#
|
||||
##################################################################################################
|
||||
#
|
||||
def GenerateSM70_Simt_complex(manifest, args):
|
||||
math_instructions = [
|
||||
MathInstruction( \
|
||||
[1, 1, 1], \
|
||||
DataType.f32, DataType.f32, DataType.f32, \
|
||||
OpcodeClass.Simt, \
|
||||
MathOperation.multiply_add_complex),
|
||||
]
|
||||
|
||||
min_cc = 70
|
||||
max_cc = 1024
|
||||
|
||||
alignment_constraints = [1,]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
tile_descriptions = [
|
||||
TileDescription([128, 128, 8], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 32], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 32], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 32], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([64, 32, 16], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([32, 64, 16], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([32, 32, 16], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
data_type = [
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32
|
||||
]
|
||||
|
||||
complex_transforms = [
|
||||
(ComplexTransform.none, ComplexTransform.none),
|
||||
(ComplexTransform.conj, ComplexTransform.none),
|
||||
(ComplexTransform.none, ComplexTransform.conj),
|
||||
(ComplexTransform.conj, ComplexTransform.conj)
|
||||
]
|
||||
|
||||
#
|
||||
|
||||
def GenerateSM70(manifest, args):
|
||||
GenerateSM70_TensorOp_884(manifest, args)
|
||||
GenerateSM70_PlanarComplexTensorOp_884(manifest, args)
|
||||
GenerateSM70_Simt_complex(manifest, args)
|
||||
|
||||
# To limit build size, WMMA GEMMs are disabled for now.
|
||||
#
|
||||
@@ -607,6 +713,9 @@ def GenerateSM75_TensorOp_1688(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 8)
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
if math_inst.element_a != math_inst.element_accumulator:
|
||||
|
||||
@@ -620,6 +729,8 @@ def GenerateSM75_TensorOp_1688(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints)
|
||||
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type_mixed, 8)
|
||||
|
||||
#
|
||||
|
||||
#
|
||||
@@ -738,6 +849,10 @@ def GenerateSM75_TensorOp_8816_TN(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
if math_inst.element_a != math_inst.element_accumulator:
|
||||
|
||||
@@ -753,6 +868,9 @@ def GenerateSM75_TensorOp_8816_TN(manifest, args):
|
||||
operations += CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
if op.tile_description.threadblock_shape[1] >= 128:
|
||||
op.C.alignment = 16
|
||||
@@ -794,6 +912,8 @@ def GenerateSM75_TensorOp_8816_Interleaved(manifest, args):
|
||||
TileDescription([256, 128, 64], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 64], 2, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 64], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 64], 2, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 64], 2, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 64], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 64], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 64], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -809,9 +929,13 @@ def GenerateSM75_TensorOp_8816_Interleaved(manifest, args):
|
||||
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
# conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
|
||||
#
|
||||
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
# data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
op.C.alignment = 8
|
||||
|
||||
#
|
||||
|
||||
#
|
||||
@@ -862,6 +986,10 @@ def GenerateSM75_TensorOp_8832_TN(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
if math_inst.element_a != math_inst.element_accumulator:
|
||||
|
||||
@@ -877,6 +1005,9 @@ def GenerateSM75_TensorOp_8832_TN(manifest, args):
|
||||
operations += CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
if op.tile_description.threadblock_shape[1] >= 128:
|
||||
op.C.alignment = 8
|
||||
@@ -920,9 +1051,9 @@ def GenerateSM75_TensorOp_8832_Interleaved(manifest, args):
|
||||
TileDescription([256, 128, 128], 2, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 128], 2, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 128], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 128], 2, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 128], 2, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 128], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 128], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 128], 2, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
@@ -938,9 +1069,13 @@ def GenerateSM75_TensorOp_8832_Interleaved(manifest, args):
|
||||
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
# conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
|
||||
#
|
||||
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
# data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
op.C.alignment = 16
|
||||
|
||||
#
|
||||
|
||||
#
|
||||
@@ -1074,6 +1209,8 @@ def GenerateSM75_Simt_complex(manifest, args):
|
||||
(ComplexTransform.conj, ComplexTransform.conj)
|
||||
]
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 1)
|
||||
#
|
||||
|
||||
def GenerateSM75(manifest, args):
|
||||
@@ -1124,6 +1261,7 @@ def GenerateSM80_TensorOp_16816(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [8, 4, 2]
|
||||
|
||||
@@ -1137,10 +1275,10 @@ def GenerateSM80_TensorOp_16816(manifest, args):
|
||||
TileDescription([128, 64, 32], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 32], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 32], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 64], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 64], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 64], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 64], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 64], 4, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 64], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 64], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -1157,6 +1295,10 @@ def GenerateSM80_TensorOp_16816(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 8)
|
||||
CreateConv3dOperator(manifest, LayoutType.TensorNDHWC, tile_descriptions, data_type, 8)
|
||||
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
if math_inst.element_a != math_inst.element_accumulator:
|
||||
|
||||
@@ -1170,6 +1312,8 @@ def GenerateSM80_TensorOp_16816(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints)
|
||||
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type_mixed, 8)
|
||||
CreateConv3dOperator(manifest, LayoutType.TensorNDHWC, tile_descriptions, data_type_mixed, 8)
|
||||
#
|
||||
|
||||
#
|
||||
@@ -1205,22 +1349,23 @@ def GenerateSM80_SparseTensorOp_16832(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [8, 4, 2]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
tile_descriptions = [
|
||||
TileDescription([256, 128, 64], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 64], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 64], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 64], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 64], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 64], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 64], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 128], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 128], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 128], 3, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 128], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
@@ -1348,6 +1493,7 @@ def GenerateSM80_TensorOp_16832_TN(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [16,]
|
||||
|
||||
@@ -1361,10 +1507,10 @@ def GenerateSM80_TensorOp_16832_TN(manifest, args):
|
||||
TileDescription([128, 64, 64], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 64], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 64], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 128], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 128], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 128], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 128], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 128], 4, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 128], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 128], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -1382,6 +1528,13 @@ def GenerateSM80_TensorOp_16832_TN(manifest, args):
|
||||
operations += CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
if op.tile_description.threadblock_shape[1] >= 128:
|
||||
op.C.alignment = 16
|
||||
@@ -1409,21 +1562,22 @@ def GenerateSM80_SparseTensorOp_16864_TN(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [16,]
|
||||
|
||||
tile_descriptions = [
|
||||
TileDescription([256, 128, 128], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 128], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 128], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 128], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 256], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 128], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 128], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 128], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 128], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 128], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 64, 128], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 256], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 64, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
|
||||
@@ -1489,10 +1643,14 @@ def GenerateSM80_TensorOp_16832_Interleaved(manifest, args):
|
||||
|
||||
operations = CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
|
||||
# conv_layout = (LayoutType.TensorNC32HW32, LayoutType.TensorC32RSK32, LayoutType.TensorNC32HW32)
|
||||
#
|
||||
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
# data_type_mixed, 16, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
op.C.alignment = 8
|
||||
|
||||
#
|
||||
|
||||
#
|
||||
@@ -1520,6 +1678,7 @@ def GenerateSM80_TensorOp_16864_TN(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [32,]
|
||||
|
||||
@@ -1533,14 +1692,14 @@ def GenerateSM80_TensorOp_16864_TN(manifest, args):
|
||||
TileDescription([128, 64, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 128], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 256], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 256], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 256], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 256], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 256], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 256], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 256], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 256], 4, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 256], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 256], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 256], 5, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 256], 5, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
]
|
||||
|
||||
data_type = [math_inst.element_a, math_inst.element_b, math_inst.element_accumulator, DataType.s32]
|
||||
@@ -1582,20 +1741,21 @@ def GenerateSM80_SparseTensorOp_168128_TN(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [32,]
|
||||
|
||||
tile_descriptions = [
|
||||
TileDescription([256, 128, 256], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 256], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 256], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 256], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 256], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 256], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 512], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 512], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 256], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 256], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 256], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 256], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 256], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 256], 6, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 512], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 512], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 512], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 512], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
@@ -1655,9 +1815,7 @@ def GenerateSM80_TensorOp_16864_Interleaved(manifest, args):
|
||||
TileDescription([256, 64, 128], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 128], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 128], 5, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 128], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 128], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
|
||||
data_type_mixed = [math_inst.element_a, math_inst.element_b, math_inst.element_a, DataType.f32]
|
||||
@@ -1666,7 +1824,12 @@ def GenerateSM80_TensorOp_16864_Interleaved(manifest, args):
|
||||
|
||||
operations += CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints, None, EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
|
||||
# conv_layout = (LayoutType.TensorNC64HW64, LayoutType.TensorC64RSK64, LayoutType.TensorNC64HW64)
|
||||
#
|
||||
# operations += CreateConv2dOperator(manifest, conv_layout, tile_descriptions,
|
||||
# data_type_mixed, 32, [ConvKind.Fprop], EpilogueFunctor.LinearCombinationClamp)
|
||||
|
||||
for op in operations:
|
||||
op.C.alignment = 16
|
||||
#
|
||||
@@ -1744,6 +1907,7 @@ def GenerateSM80_TensorOp_1688(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [4, 2, 1]
|
||||
|
||||
@@ -1757,11 +1921,11 @@ def GenerateSM80_TensorOp_1688(manifest, args):
|
||||
TileDescription([128, 64, 16], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 16], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 16], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 32], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 32], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 32], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 32], 4, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 32], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 32], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([64, 128, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 32], 5, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -1787,6 +1951,10 @@ def GenerateSM80_TensorOp_1688(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type_mixed, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 4)
|
||||
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type_mixed, 4)
|
||||
#
|
||||
|
||||
#
|
||||
@@ -1822,6 +1990,7 @@ def GenerateSM80_TensorOp_1688_fast_math(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [4, 2, 1]
|
||||
|
||||
@@ -1835,11 +2004,11 @@ def GenerateSM80_TensorOp_1688_fast_math(manifest, args):
|
||||
TileDescription([128, 64, 16], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 16], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 16], 10, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 32], 4, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 32], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 32], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 32], 4, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 256, 32], 4, [1, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 128, 32], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 32], 5, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -1850,6 +2019,8 @@ def GenerateSM80_TensorOp_1688_fast_math(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 4)
|
||||
#
|
||||
|
||||
#
|
||||
@@ -1875,22 +2046,23 @@ def GenerateSM80_SparseTensorOp_16816_fast_math(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [4, 2, 1]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
tile_descriptions = [
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 128, 32], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 256, 32], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 32], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 256, 32], 4, [1, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 32], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 32], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 32], 6, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([256, 64, 64], 3, [4, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 64], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 64], 3, [2, 4, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([256, 64, 64], 3, [4, 1, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 64], 4, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([ 64, 128, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 64], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
@@ -1971,13 +2143,14 @@ def GenerateSM80_TensorOp_884(manifest, args):
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
max_cc_smem_limited = 80
|
||||
|
||||
alignment_constraints = [1,]
|
||||
|
||||
tile_descriptions = [
|
||||
TileDescription([128, 128, 16], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([64, 128, 16], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 16], 3, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 16], 3, [4, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([64, 128, 16], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([128, 64, 16], 3, [2, 2, 1], math_inst, min_cc, max_cc_smem_limited),
|
||||
TileDescription([64, 64, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([64, 32, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([32, 64, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -2090,7 +2263,7 @@ def GenerateSM80_TensorOp_884_complex_gaussian(manifest, args):
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
def GenerateSM80_Simt(manifest, args):
|
||||
def GenerateSM80_Simt_f32(manifest, args):
|
||||
layouts = [
|
||||
(LayoutType.ColumnMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.ColumnMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
@@ -2136,8 +2309,55 @@ def GenerateSM80_Simt(manifest, args):
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 1)
|
||||
#
|
||||
|
||||
|
||||
#
|
||||
def GenerateSM80_Simt_f64(manifest, args):
|
||||
layouts = [
|
||||
(LayoutType.ColumnMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.ColumnMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
]
|
||||
|
||||
math_instructions = [
|
||||
MathInstruction( \
|
||||
[1, 1, 1], \
|
||||
DataType.f64, DataType.f64, DataType.f64, \
|
||||
OpcodeClass.Simt, \
|
||||
MathOperation.multiply_add),
|
||||
]
|
||||
|
||||
min_cc = 80
|
||||
max_cc = 1024
|
||||
|
||||
alignment_constraints = [1,]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
tile_descriptions = [
|
||||
TileDescription([128, 128, 8], 3, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 64, 8], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 128, 8], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 64, 64, 8], 5, [2, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 32, 8], 5, [2, 1, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([ 32, 128, 8], 5, [1, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
|
||||
data_type = [
|
||||
math_inst.element_a,
|
||||
math_inst.element_b,
|
||||
math_inst.element_accumulator,
|
||||
math_inst.element_accumulator,
|
||||
]
|
||||
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, \
|
||||
data_type, alignment_constraints)
|
||||
#
|
||||
|
||||
|
||||
##################################################################################################
|
||||
#
|
||||
def GenerateSM80_Simt_complex(manifest, args):
|
||||
@@ -2154,7 +2374,29 @@ def GenerateSM80_Simt_complex(manifest, args):
|
||||
|
||||
alignment_constraints = [1,]
|
||||
|
||||
data_type = [
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32
|
||||
]
|
||||
|
||||
layouts = [
|
||||
(LayoutType.ColumnMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.ColumnMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.ColumnMajor, LayoutType.ColumnMajor),
|
||||
(LayoutType.RowMajor, LayoutType.RowMajor, LayoutType.ColumnMajor),
|
||||
]
|
||||
|
||||
complex_transforms = [
|
||||
(ComplexTransform.none, ComplexTransform.none),
|
||||
(ComplexTransform.conj, ComplexTransform.none),
|
||||
(ComplexTransform.none, ComplexTransform.conj),
|
||||
(ComplexTransform.conj, ComplexTransform.conj)
|
||||
]
|
||||
|
||||
for math_inst in math_instructions:
|
||||
|
||||
tile_descriptions = [
|
||||
TileDescription([128, 128, 8], 5, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([128, 128, 8], 4, [4, 2, 1], math_inst, min_cc, max_cc),
|
||||
@@ -2165,20 +2407,11 @@ def GenerateSM80_Simt_complex(manifest, args):
|
||||
TileDescription([32, 64, 16], 4, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
TileDescription([32, 32, 16], 5, [2, 2, 1], math_inst, min_cc, max_cc),
|
||||
]
|
||||
data_type = [
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32,
|
||||
DataType.cf32
|
||||
]
|
||||
|
||||
complex_transforms = [
|
||||
(ComplexTransform.none, ComplexTransform.none),
|
||||
(ComplexTransform.conj, ComplexTransform.none),
|
||||
(ComplexTransform.none, ComplexTransform.conj),
|
||||
(ComplexTransform.conj, ComplexTransform.conj)
|
||||
]
|
||||
CreateGemmOperator(manifest, layouts, tile_descriptions, data_type, alignment_constraints, complex_transforms)
|
||||
|
||||
conv_layout = (LayoutType.TensorNHWC, LayoutType.TensorNHWC, LayoutType.TensorNHWC)
|
||||
CreateConv2dOperator(manifest, conv_layout, tile_descriptions, data_type, 1)
|
||||
#
|
||||
|
||||
###################################################################################################
|
||||
@@ -2202,7 +2435,8 @@ def GenerateSM80(manifest, args):
|
||||
GenerateSM80_SparseTensorOp_168128_TN(manifest, args)
|
||||
GenerateSM80_TensorOp_16864_Interleaved(manifest, args)
|
||||
GenerateSM80_TensorOp_168256(manifest, args)
|
||||
GenerateSM80_Simt(manifest, args)
|
||||
GenerateSM80_Simt_f32(manifest, args)
|
||||
GenerateSM80_Simt_f64(manifest, args)
|
||||
GenerateSM80_Simt_complex(manifest, args)
|
||||
|
||||
###################################################################################################
|
||||
|
||||
@@ -71,6 +71,7 @@ class DataType(enum.Enum):
|
||||
cu16 = enum_auto()
|
||||
cu32 = enum_auto()
|
||||
cu64 = enum_auto()
|
||||
invalid = enum_auto()
|
||||
|
||||
#
|
||||
ShortDataTypeNames = {
|
||||
@@ -260,6 +261,8 @@ MathOperationTag = {
|
||||
class LayoutType(enum.Enum):
|
||||
ColumnMajor = enum_auto()
|
||||
RowMajor = enum_auto()
|
||||
ColumnMajorInterleaved2 = enum_auto()
|
||||
RowMajorInterleaved2 = enum_auto()
|
||||
ColumnMajorInterleaved32 = enum_auto()
|
||||
RowMajorInterleaved32 = enum_auto()
|
||||
ColumnMajorInterleaved64 = enum_auto()
|
||||
@@ -268,13 +271,17 @@ class LayoutType(enum.Enum):
|
||||
TensorNDHWC = enum_auto()
|
||||
TensorNCHW = enum_auto()
|
||||
TensorNGHWC = enum_auto()
|
||||
TensorNCxHW32 = enum_auto()
|
||||
TensorNCxHW64 = enum_auto()
|
||||
TensorNC32HW32 = enum_auto()
|
||||
TensorNC64HW64 = enum_auto()
|
||||
TensorC32RSK32 = enum_auto()
|
||||
TensorC64RSK64 = enum_auto()
|
||||
|
||||
#
|
||||
LayoutTag = {
|
||||
LayoutType.ColumnMajor: 'cutlass::layout::ColumnMajor',
|
||||
LayoutType.RowMajor: 'cutlass::layout::RowMajor',
|
||||
LayoutType.ColumnMajorInterleaved2: 'cutlass::layout::ColumnMajorInterleaved<2>',
|
||||
LayoutType.RowMajorInterleaved2: 'cutlass::layout::RowMajorInterleaved<2>',
|
||||
LayoutType.ColumnMajorInterleaved32: 'cutlass::layout::ColumnMajorInterleaved<32>',
|
||||
LayoutType.RowMajorInterleaved32: 'cutlass::layout::RowMajorInterleaved<32>',
|
||||
LayoutType.ColumnMajorInterleaved64: 'cutlass::layout::ColumnMajorInterleaved<64>',
|
||||
@@ -283,14 +290,18 @@ LayoutTag = {
|
||||
LayoutType.TensorNDHWC: 'cutlass::layout::TensorNDHWC',
|
||||
LayoutType.TensorNCHW: 'cutlass::layout::TensorNCHW',
|
||||
LayoutType.TensorNGHWC: 'cutlass::layout::TensorNGHWC',
|
||||
LayoutType.TensorNCxHW32: 'cutlass::layout::TensorNCxHW32',
|
||||
LayoutType.TensorNCxHW64: 'cutlass::layout::TensorNCxHW64'
|
||||
LayoutType.TensorNC32HW32: 'cutlass::layout::TensorNCxHWx<32>',
|
||||
LayoutType.TensorC32RSK32: 'cutlass::layout::TensorCxRSKx<32>',
|
||||
LayoutType.TensorNC64HW64: 'cutlass::layout::TensorNCxHWx<64>',
|
||||
LayoutType.TensorC64RSK64: 'cutlass::layout::TensorCxRSKx<64>',
|
||||
}
|
||||
|
||||
#
|
||||
TransposedLayout = {
|
||||
LayoutType.ColumnMajor: LayoutType.RowMajor,
|
||||
LayoutType.RowMajor: LayoutType.ColumnMajor,
|
||||
LayoutType.ColumnMajorInterleaved2: LayoutType.RowMajorInterleaved2,
|
||||
LayoutType.RowMajorInterleaved2: LayoutType.ColumnMajorInterleaved2,
|
||||
LayoutType.ColumnMajorInterleaved32: LayoutType.RowMajorInterleaved32,
|
||||
LayoutType.RowMajorInterleaved32: LayoutType.ColumnMajorInterleaved32,
|
||||
LayoutType.ColumnMajorInterleaved64: LayoutType.RowMajorInterleaved64,
|
||||
@@ -301,17 +312,21 @@ TransposedLayout = {
|
||||
#
|
||||
ShortLayoutTypeNames = {
|
||||
LayoutType.ColumnMajor: 'n',
|
||||
LayoutType.ColumnMajorInterleaved32: 'n2',
|
||||
LayoutType.ColumnMajorInterleaved32: 'n32',
|
||||
LayoutType.ColumnMajorInterleaved64: 'n64',
|
||||
LayoutType.RowMajor: 't',
|
||||
LayoutType.RowMajorInterleaved2: 't2',
|
||||
LayoutType.RowMajorInterleaved32: 't32',
|
||||
LayoutType.RowMajorInterleaved64: 't64',
|
||||
LayoutType.TensorNHWC: 'nhwc',
|
||||
LayoutType.TensorNDHWC: 'ndhwc',
|
||||
LayoutType.TensorNCHW: 'nchw',
|
||||
LayoutType.TensorNGHWC: 'nghwc',
|
||||
LayoutType.TensorNCxHW32: 'ncxhw32',
|
||||
LayoutType.TensorNCxHW64: 'ncxhw64'
|
||||
LayoutType.TensorNC32HW32: 'nc32hw32',
|
||||
LayoutType.TensorNC64HW64: 'nc64hw64',
|
||||
LayoutType.TensorC32RSK32: 'c32rsk32',
|
||||
LayoutType.TensorC64RSK64: 'c64rsk64'
|
||||
}
|
||||
|
||||
#
|
||||
@@ -346,9 +361,14 @@ OpcodeClassTag = {
|
||||
#
|
||||
class OperationKind(enum.Enum):
|
||||
Gemm = enum_auto()
|
||||
Conv2d = enum_auto()
|
||||
Conv3d = enum_auto()
|
||||
|
||||
#
|
||||
OperationKindNames = {
|
||||
OperationKind.Gemm: 'gemm'
|
||||
, OperationKind.Conv2d: 'conv2d'
|
||||
, OperationKind.Conv3d: 'conv3d'
|
||||
}
|
||||
|
||||
#
|
||||
@@ -424,6 +444,61 @@ SwizzlingFunctorTag = {
|
||||
SwizzlingFunctor.Identity4: 'cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<4>',
|
||||
SwizzlingFunctor.Identity8: 'cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>',
|
||||
}
|
||||
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
class ConvKind(enum.Enum):
|
||||
Fprop = enum_auto()
|
||||
Dgrad = enum_auto()
|
||||
Wgrad = enum_auto()
|
||||
|
||||
#
|
||||
ConvKindTag = {
|
||||
ConvKind.Fprop: 'cutlass::conv::Operator::kFprop',
|
||||
ConvKind.Dgrad: 'cutlass::conv::Operator::kDgrad',
|
||||
ConvKind.Wgrad: 'cutlass::conv::Operator::kWgrad'
|
||||
}
|
||||
|
||||
ConvKindNames = {
|
||||
ConvKind.Fprop: 'fprop',
|
||||
ConvKind.Dgrad: 'dgrad',
|
||||
ConvKind.Wgrad: 'wgrad',
|
||||
}
|
||||
|
||||
#
|
||||
class IteratorAlgorithm(enum.Enum):
|
||||
Analytic = enum_auto()
|
||||
Optimized = enum_auto()
|
||||
|
||||
#
|
||||
IteratorAlgorithmTag = {
|
||||
IteratorAlgorithm.Analytic: 'cutlass::conv::IteratorAlgorithm::kAnalytic',
|
||||
IteratorAlgorithm.Optimized: 'cutlass::conv::IteratorAlgorithm::kOptimized',
|
||||
}
|
||||
|
||||
IteratorAlgorithmNames = {
|
||||
IteratorAlgorithm.Analytic: 'analytic',
|
||||
IteratorAlgorithm.Optimized: 'optimized',
|
||||
}
|
||||
|
||||
#
|
||||
class StrideSupport(enum.Enum):
|
||||
Strided = enum_auto()
|
||||
Unity = enum_auto()
|
||||
|
||||
#
|
||||
StrideSupportTag = {
|
||||
StrideSupport.Strided: 'cutlass::conv::StrideSupport::kStrided',
|
||||
StrideSupport.Unity: 'cutlass::conv::StrideSupport::kUnity',
|
||||
}
|
||||
|
||||
StrideSupportNames = {
|
||||
StrideSupport.Strided: '',
|
||||
StrideSupport.Unity: 'unity_stride',
|
||||
}
|
||||
|
||||
|
||||
###################################################################################################
|
||||
|
||||
#
|
||||
|
||||
@@ -10,6 +10,9 @@ import shutil
|
||||
|
||||
from library import *
|
||||
from gemm_operation import *
|
||||
from conv2d_operation import *
|
||||
from conv3d_operation import *
|
||||
|
||||
###################################################################################################
|
||||
|
||||
class EmitOperationKindLibrary:
|
||||
@@ -20,6 +23,8 @@ class EmitOperationKindLibrary:
|
||||
|
||||
self.emitters = {
|
||||
OperationKind.Gemm: EmitGemmConfigurationLibrary
|
||||
, OperationKind.Conv2d: EmitConv2dConfigurationLibrary
|
||||
, OperationKind.Conv3d: EmitConv3dConfigurationLibrary
|
||||
}
|
||||
|
||||
self.configurations = [];
|
||||
@@ -112,7 +117,10 @@ class Manifest:
|
||||
def __init__(self, args):
|
||||
self.operations = {}
|
||||
self.args = args
|
||||
self.compute_capabilities = [int(x) for x in args.architectures.split(';')]
|
||||
|
||||
architectures = args.architectures.split(';') if len(args.architectures) else ['50',]
|
||||
self.compute_capabilities = [int(x) for x in architectures]
|
||||
|
||||
self.selected_kernels = []
|
||||
|
||||
if args.operations == 'all':
|
||||
@@ -121,6 +129,8 @@ class Manifest:
|
||||
|
||||
operations_list = [
|
||||
OperationKind.Gemm
|
||||
, OperationKind.Conv2d
|
||||
, OperationKind.Conv3d
|
||||
]
|
||||
|
||||
self.operations_enabled = [x for x in operations_list if OperationKindNames[x] in args.operations.split(',')]
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for all CONV operation kinds in CUTLASS Library.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/conv/kernel/default_conv2d_fprop.h"
|
||||
#include "cutlass/conv/kernel/default_conv2d_dgrad.h"
|
||||
#include "cutlass/conv/kernel/default_conv2d_wgrad.h"
|
||||
#include "cutlass/conv/device/implicit_gemm_convolution.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "library_internal.h"
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
|
||||
#include "cutlass/util/reference/host/convolution.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/core_io.h"
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Operator_>
|
||||
class Conv2dOperationBase : public Operation {
|
||||
public:
|
||||
|
||||
using Operator = Operator_;
|
||||
|
||||
using ElementA = typename Operator::ElementA;
|
||||
using LayoutA = typename Operator::LayoutA;
|
||||
using ElementB = typename Operator::ElementB;
|
||||
using LayoutB = typename Operator::LayoutB;
|
||||
using ElementC = typename Operator::ElementC;
|
||||
using LayoutC = typename Operator::LayoutC;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
|
||||
static cutlass::conv::IteratorAlgorithm const kIteratorAlgorithm = Operator::kIteratorAlgorithm;
|
||||
static cutlass::conv::Operator const kConvolutionalOperator = Operator::kConvolutionalOperator;
|
||||
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
|
||||
protected:
|
||||
|
||||
///
|
||||
ConvDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
Conv2dOperationBase(char const *name = "unknown_conv2d") {
|
||||
|
||||
description_.name = name;
|
||||
description_.provider = Provider::kCUTLASS;
|
||||
description_.kind = OperationKind::kConv2d;
|
||||
description_.conv_dim = Operator::kConvDim;
|
||||
|
||||
description_.iterator_algorithm = IteratorAlgorithmMap<Operator::kIteratorAlgorithm>::kId;
|
||||
|
||||
description_.tile_description.threadblock_shape = make_Coord(
|
||||
Operator::ThreadblockShape::kM,
|
||||
Operator::ThreadblockShape::kN,
|
||||
Operator::ThreadblockShape::kK);
|
||||
|
||||
description_.tile_description.threadblock_stages = Operator::kStages;
|
||||
|
||||
description_.tile_description.warp_count = make_Coord(
|
||||
Operator::ImplicitGemmKernel::WarpCount::kM,
|
||||
Operator::ImplicitGemmKernel::WarpCount::kN,
|
||||
Operator::ImplicitGemmKernel::WarpCount::kK);
|
||||
|
||||
description_.tile_description.math_instruction.instruction_shape = make_Coord(
|
||||
Operator::InstructionShape::kM,
|
||||
Operator::InstructionShape::kN,
|
||||
Operator::InstructionShape::kK);
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.opcode_class =
|
||||
OpcodeClassMap<typename Operator::OperatorClass>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.math_operation =
|
||||
MathOperationMap<typename Operator::MathOperator>::kId;
|
||||
|
||||
description_.tile_description.minimum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMin;
|
||||
|
||||
description_.tile_description.maximum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMax;
|
||||
|
||||
description_.A = make_TensorDescription<ElementA, LayoutA>();
|
||||
description_.B = make_TensorDescription<ElementB, LayoutB>();
|
||||
description_.C = make_TensorDescription<ElementC, LayoutC>();
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
// TODO: Add split k mode Serial and parallel to convolutions
|
||||
// description_.split_k_mode = Operator::kSplitK ? SplitKMode::kSerial : SplitKMode::kNone;
|
||||
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Conv2d library operation class for cutlass profiler
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename Operator_>
|
||||
class Conv2dOperation : public Conv2dOperationBase<Operator_> {
|
||||
public:
|
||||
|
||||
using Operator = Operator_;
|
||||
|
||||
using ElementA = typename Operator::ElementA;
|
||||
using LayoutA = typename Operator::LayoutA;
|
||||
using ElementB = typename Operator::ElementB;
|
||||
using LayoutB = typename Operator::LayoutB;
|
||||
using ElementC = typename Operator::ElementC;
|
||||
using LayoutC = typename Operator::LayoutC;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
|
||||
static cutlass::conv::Operator const kConvolutionalOperator = Operator::kConvolutionalOperator;
|
||||
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
|
||||
public:
|
||||
/// Constructor
|
||||
Conv2dOperation(char const *name = "unknown_conv2d_fprop") : Conv2dOperationBase<Operator_>(name) {
|
||||
this->description_.conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status construct_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
Conv2dConfiguration const *configuration) {
|
||||
|
||||
|
||||
operator_args.problem_size = configuration->problem_size;
|
||||
|
||||
operator_args.ref_A =
|
||||
{
|
||||
nullptr,
|
||||
LayoutA::packed(implicit_gemm_tensor_a_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_B =
|
||||
{
|
||||
nullptr,
|
||||
LayoutB::packed(implicit_gemm_tensor_b_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_C =
|
||||
{
|
||||
nullptr,
|
||||
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_D =
|
||||
{
|
||||
nullptr,
|
||||
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.split_k_mode = configuration->split_k_mode;
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status update_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
ConvArguments const *arguments) {
|
||||
|
||||
if (arguments->pointer_mode == ScalarPointerMode::kHost) {
|
||||
typename Operator::EpilogueOutputOp::Params params(
|
||||
*static_cast<ElementCompute const *>(arguments->alpha),
|
||||
*static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output_op = params;
|
||||
}
|
||||
else if (arguments->pointer_mode == ScalarPointerMode::kDevice){
|
||||
typename Operator::EpilogueOutputOp::Params params(
|
||||
static_cast<ElementCompute const *>(arguments->alpha),
|
||||
static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output_op = params;
|
||||
}
|
||||
else {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
operator_args.ref_A.reset(static_cast<ElementA *>(const_cast<void *>(arguments->A)));
|
||||
operator_args.ref_B.reset(static_cast<ElementB *>(const_cast<void *>(arguments->B)));
|
||||
operator_args.ref_C.reset(static_cast<ElementC *>(const_cast<void *>(arguments->C)));
|
||||
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
/// Returns success if the operation can proceed
|
||||
virtual Status can_implement(
|
||||
void const *configuration_ptr,
|
||||
void const *arguments_ptr) const {
|
||||
|
||||
Conv2dConfiguration const *configuration =
|
||||
static_cast<Conv2dConfiguration const *>(configuration_ptr);
|
||||
|
||||
ConvArguments const *arguments =
|
||||
static_cast<ConvArguments const *>(arguments_ptr);
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(args, configuration);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
status = update_arguments_(args, arguments);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
return Operator::can_implement(args);
|
||||
|
||||
}
|
||||
|
||||
/// Gets the host-side workspace
|
||||
virtual uint64_t get_host_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
return sizeof(Operator);
|
||||
}
|
||||
|
||||
/// Gets the device-side workspace
|
||||
virtual uint64_t get_device_workspace_size(
|
||||
void const *configuration_ptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<Conv2dConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return Operator::get_workspace_size(args);
|
||||
}
|
||||
|
||||
/// Initializes the workspace
|
||||
virtual Status initialize(
|
||||
void const *configuration_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<Conv2dConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = new (host_workspace) Operator;
|
||||
//std::cout << "initialize library::Conv2dOperation" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->initialize(args, device_workspace, stream);
|
||||
|
||||
}
|
||||
|
||||
/// Runs the kernel
|
||||
virtual Status run(
|
||||
void const *arguments_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = update_arguments_(
|
||||
args,
|
||||
static_cast<ConvArguments const *>(arguments_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = static_cast<Operator *>(host_workspace);
|
||||
|
||||
status = op->update(args, device_workspace);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
//std::cout << "run library::Conv2dOperation" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->run(stream);
|
||||
}
|
||||
|
||||
/// Call print_operator_args from the Conv2dOperation::initialize()
|
||||
// to dump arguments passed on to cutlass operator for debugging
|
||||
void print_operator_args(OperatorArguments &operator_args) const {
|
||||
std::cout << "Conv2dOperation::OperatorArguments" << std::endl
|
||||
<< " problem_size:" << std::endl
|
||||
<< operator_args.problem_size << std::endl
|
||||
<< " split_k_mode: "
|
||||
<< (operator_args.split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial" : "parallel") << std::endl
|
||||
<< " epilouge (alpha, beta): "
|
||||
<< operator_args.output_op.alpha << ", "
|
||||
<< operator_args.output_op.beta << std::endl
|
||||
<< " ref_A (ptr, {stride}): "
|
||||
<< operator_args.ref_A.data() << ", {"
|
||||
<< operator_args.ref_A.stride(0) << ", "
|
||||
<< operator_args.ref_A.stride(1) << ", "
|
||||
<< operator_args.ref_A.stride(2) << "}" << std::endl
|
||||
<< " ref_B (ptr, {stride}): "
|
||||
<< operator_args.ref_B.data() << ", {"
|
||||
<< operator_args.ref_B.stride(0) << ", "
|
||||
<< operator_args.ref_B.stride(1) << ", "
|
||||
<< operator_args.ref_B.stride(2) << "}" << std::endl
|
||||
<< " ref_C (ptr, {stride}): "
|
||||
<< operator_args.ref_C.data() << ", {"
|
||||
<< operator_args.ref_C.stride(0) << ", "
|
||||
<< operator_args.ref_C.stride(1) << ", "
|
||||
<< operator_args.ref_C.stride(2) << "}" << std::endl
|
||||
<< " ref_D (ptr, {stride}): "
|
||||
<< operator_args.ref_D.data() << ", {"
|
||||
<< operator_args.ref_D.stride(0) << ", "
|
||||
<< operator_args.ref_D.stride(1) << ", "
|
||||
<< operator_args.ref_D.stride(2) << "}" << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,378 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for all CONV operation kinds in CUTLASS Library.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/conv/kernel/default_conv3d_fprop.h"
|
||||
#include "cutlass/conv/kernel/default_conv3d_dgrad.h"
|
||||
#include "cutlass/conv/kernel/default_conv3d_wgrad.h"
|
||||
#include "cutlass/conv/device/implicit_gemm_convolution.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "library_internal.h"
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
|
||||
#include "cutlass/util/reference/host/convolution.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/core_io.h"
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Operator_>
|
||||
class Conv3dOperationBase : public Operation {
|
||||
public:
|
||||
|
||||
using Operator = Operator_;
|
||||
|
||||
using ElementA = typename Operator::ElementA;
|
||||
using LayoutA = typename Operator::LayoutA;
|
||||
using ElementB = typename Operator::ElementB;
|
||||
using LayoutB = typename Operator::LayoutB;
|
||||
using ElementC = typename Operator::ElementC;
|
||||
using LayoutC = typename Operator::LayoutC;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
|
||||
static cutlass::conv::IteratorAlgorithm const kIteratorAlgorithm = Operator::kIteratorAlgorithm;
|
||||
static cutlass::conv::Operator const kConvolutionalOperator = Operator::kConvolutionalOperator;
|
||||
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
|
||||
protected:
|
||||
|
||||
///
|
||||
ConvDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
Conv3dOperationBase(char const *name = "unknown_conv3d") {
|
||||
|
||||
description_.name = name;
|
||||
description_.provider = Provider::kCUTLASS;
|
||||
description_.kind = OperationKind::kConv3d;
|
||||
description_.conv_dim = Operator::kConvDim;
|
||||
|
||||
description_.iterator_algorithm = IteratorAlgorithmMap<Operator::kIteratorAlgorithm>::kId;
|
||||
|
||||
description_.tile_description.threadblock_shape = make_Coord(
|
||||
Operator::ThreadblockShape::kM,
|
||||
Operator::ThreadblockShape::kN,
|
||||
Operator::ThreadblockShape::kK);
|
||||
|
||||
description_.tile_description.threadblock_stages = Operator::kStages;
|
||||
|
||||
description_.tile_description.warp_count = make_Coord(
|
||||
Operator::ImplicitGemmKernel::WarpCount::kM,
|
||||
Operator::ImplicitGemmKernel::WarpCount::kN,
|
||||
Operator::ImplicitGemmKernel::WarpCount::kK);
|
||||
|
||||
description_.tile_description.math_instruction.instruction_shape = make_Coord(
|
||||
Operator::InstructionShape::kM,
|
||||
Operator::InstructionShape::kN,
|
||||
Operator::InstructionShape::kK);
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.opcode_class =
|
||||
OpcodeClassMap<typename Operator::OperatorClass>::kId;
|
||||
|
||||
description_.tile_description.minimum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMin;
|
||||
|
||||
description_.tile_description.maximum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMax;
|
||||
|
||||
description_.A = make_TensorDescription<ElementA, LayoutA>();
|
||||
description_.B = make_TensorDescription<ElementB, LayoutB>();
|
||||
description_.C = make_TensorDescription<ElementC, LayoutC>();
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Conv2d library operation class for cutlass profiler
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename Operator_>
|
||||
class Conv3dOperation : public Conv3dOperationBase<Operator_> {
|
||||
public:
|
||||
|
||||
using Operator = Operator_;
|
||||
|
||||
using ElementA = typename Operator::ElementA;
|
||||
using LayoutA = typename Operator::LayoutA;
|
||||
using ElementB = typename Operator::ElementB;
|
||||
using LayoutB = typename Operator::LayoutB;
|
||||
using ElementC = typename Operator::ElementC;
|
||||
using LayoutC = typename Operator::LayoutC;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
|
||||
static cutlass::conv::Operator const kConvolutionalOperator = Operator::kConvolutionalOperator;
|
||||
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
|
||||
public:
|
||||
/// Constructor
|
||||
Conv3dOperation(char const *name = "unknown_conv3d_fprop") : Conv3dOperationBase<Operator_>(name) {
|
||||
this->description_.conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status construct_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
Conv3dConfiguration const *configuration) {
|
||||
|
||||
|
||||
operator_args.problem_size = configuration->problem_size;
|
||||
|
||||
operator_args.ref_A =
|
||||
{
|
||||
nullptr,
|
||||
LayoutA::packed(implicit_gemm_tensor_a_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_B =
|
||||
{
|
||||
nullptr,
|
||||
LayoutB::packed(implicit_gemm_tensor_b_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_C =
|
||||
{
|
||||
nullptr,
|
||||
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.ref_D =
|
||||
{
|
||||
nullptr,
|
||||
LayoutC::packed(implicit_gemm_tensor_c_extent(kConvolutionalOperator, configuration->problem_size))
|
||||
};
|
||||
|
||||
operator_args.split_k_mode = configuration->split_k_mode;
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status update_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
ConvArguments const *arguments) {
|
||||
|
||||
if (arguments->pointer_mode == ScalarPointerMode::kHost) {
|
||||
typename Operator::EpilogueOutputOp::Params params(
|
||||
*static_cast<ElementCompute const *>(arguments->alpha),
|
||||
*static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output_op = params;
|
||||
}
|
||||
else if (arguments->pointer_mode == ScalarPointerMode::kDevice){
|
||||
typename Operator::EpilogueOutputOp::Params params(
|
||||
static_cast<ElementCompute const *>(arguments->alpha),
|
||||
static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output_op = params;
|
||||
}
|
||||
else {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
operator_args.ref_A.reset(static_cast<ElementA *>(const_cast<void *>(arguments->A)));
|
||||
operator_args.ref_B.reset(static_cast<ElementB *>(const_cast<void *>(arguments->B)));
|
||||
operator_args.ref_C.reset(static_cast<ElementC *>(const_cast<void *>(arguments->C)));
|
||||
operator_args.ref_D.reset(static_cast<ElementC *>(const_cast<void *>(arguments->D)));
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
/// Returns success if the operation can proceed
|
||||
virtual Status can_implement(
|
||||
void const *configuration_ptr,
|
||||
void const *arguments_ptr) const {
|
||||
|
||||
Conv3dConfiguration const *configuration =
|
||||
static_cast<Conv3dConfiguration const *>(configuration_ptr);
|
||||
|
||||
ConvArguments const *arguments =
|
||||
static_cast<ConvArguments const *>(arguments_ptr);
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(args, configuration);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
status = update_arguments_(args, arguments);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
return Operator::can_implement(args);
|
||||
|
||||
}
|
||||
|
||||
/// Gets the host-side workspace
|
||||
virtual uint64_t get_host_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
return sizeof(Operator);
|
||||
}
|
||||
|
||||
/// Gets the device-side workspace
|
||||
virtual uint64_t get_device_workspace_size(
|
||||
void const *configuration_ptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<Conv3dConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return Operator::get_workspace_size(args);
|
||||
}
|
||||
|
||||
/// Initializes the workspace
|
||||
virtual Status initialize(
|
||||
void const *configuration_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<Conv3dConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = new (host_workspace) Operator;
|
||||
//std::cout << "initialize library::Conv3dOperation" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->initialize(args, device_workspace, stream);
|
||||
|
||||
}
|
||||
|
||||
/// Runs the kernel
|
||||
virtual Status run(
|
||||
void const *arguments_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = update_arguments_(
|
||||
args,
|
||||
static_cast<ConvArguments const *>(arguments_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = static_cast<Operator *>(host_workspace);
|
||||
|
||||
status = op->update(args, device_workspace);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
//std::cout << "run library::Conv3dOperation" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->run(stream);
|
||||
}
|
||||
|
||||
/// Call print_operator_args from the Conv3dOperation::initialize()
|
||||
// to dump arguments passed on to cutlass operator for debugging
|
||||
void print_operator_args(OperatorArguments &operator_args) const {
|
||||
std::cout << "Conv3dOperation::OperatorArguments" << std::endl
|
||||
<< " problem_size: "
|
||||
<< operator_args.problem_size << std::endl
|
||||
<< " split_k_mode: "
|
||||
<< (operator_args.split_k_mode == cutlass::conv::SplitKMode::kSerial ? "serial" : "parallel") << std::endl
|
||||
<< " epilouge (alpha, beta): "
|
||||
<< operator_args.output_op.alpha << ", "
|
||||
<< operator_args.output_op.beta << std::endl
|
||||
<< " ref_A (ptr, {stride}): "
|
||||
<< operator_args.ref_A.data() << ", {"
|
||||
<< operator_args.ref_A.stride(0) << ", "
|
||||
<< operator_args.ref_A.stride(1) << ", "
|
||||
<< operator_args.ref_A.stride(2) << ", "
|
||||
<< operator_args.ref_A.stride(3) << "}" << std::endl
|
||||
<< " ref_B (ptr, {stride}): "
|
||||
<< operator_args.ref_B.data() << ", {"
|
||||
<< operator_args.ref_B.stride(0) << ", "
|
||||
<< operator_args.ref_B.stride(1) << ", "
|
||||
<< operator_args.ref_B.stride(2) << ", "
|
||||
<< operator_args.ref_B.stride(3) << "}" << std::endl
|
||||
<< " ref_C (ptr, {stride}): "
|
||||
<< operator_args.ref_C.data() << ", {"
|
||||
<< operator_args.ref_C.stride(0) << ", "
|
||||
<< operator_args.ref_C.stride(1) << ", "
|
||||
<< operator_args.ref_C.stride(2) << ", "
|
||||
<< operator_args.ref_C.stride(3) << "}" << std::endl
|
||||
<< " ref_D (ptr, {stride}): "
|
||||
<< operator_args.ref_D.data() << ", {"
|
||||
<< operator_args.ref_D.stride(0) << ", "
|
||||
<< operator_args.ref_D.stride(1) << ", "
|
||||
<< operator_args.ref_D.stride(2) << ", "
|
||||
<< operator_args.ref_D.stride(3) << "}" << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -1037,8 +1037,70 @@ Status Handle::gemm_planar_complex_array(
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Finds conv operation instances with Conv::ElementC = Reduction::ElementWorkspace
|
||||
Operation const* find_conv_operation_for_parallel_reduction(Operation const *operation) {
|
||||
|
||||
ConvDescription const &conv_desc =
|
||||
static_cast<ConvDescription const &>(operation->description());
|
||||
|
||||
// if the curren conv operation accumulator and output data type match return operation
|
||||
if(conv_desc.tile_description.math_instruction.element_accumulator == conv_desc.C.element) {
|
||||
return operation;
|
||||
}
|
||||
|
||||
// find conv operation to match conv output and reduction workspace data type
|
||||
ConvFunctionalKey key(
|
||||
library::Provider::kCUTLASS,
|
||||
conv_desc.conv_kind,
|
||||
conv_desc.A.element,
|
||||
conv_desc.A.layout,
|
||||
conv_desc.B.element,
|
||||
conv_desc.B.layout,
|
||||
conv_desc.tile_description.math_instruction.element_accumulator,
|
||||
conv_desc.C.layout,
|
||||
conv_desc.tile_description.math_instruction.element_accumulator,
|
||||
conv_desc.element_epilogue);
|
||||
|
||||
// conv operation table for conv2d or conv3d
|
||||
auto conv_operations = (conv_desc.kind == OperationKind::kConv2d) ?
|
||||
Singleton::get().operation_table.conv2d_operations :
|
||||
Singleton::get().operation_table.conv3d_operations;
|
||||
|
||||
// find ConvFunctionalKey in convolution operation table
|
||||
auto operators_it = conv_operations.find(key);
|
||||
|
||||
if (operators_it == conv_operations.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (operators_it->second.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// conv operation for same compute capability and iterator algorithm
|
||||
ConvPreferenceKey preference_key(
|
||||
conv_desc.tile_description.minimum_compute_capability,
|
||||
conv_desc.iterator_algorithm);
|
||||
|
||||
auto it = operators_it->second.find(preference_key);
|
||||
|
||||
if(it == operators_it->second.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// return matching conv opertion (same tile sizes and instruction)
|
||||
for (auto op : it->second) {
|
||||
if (op->description().tile_description == operation->description().tile_description) {
|
||||
return op;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -227,6 +227,23 @@ template <> struct LayoutMap<cutlass::layout::TensorNHWC> {
|
||||
template <> struct LayoutMap<cutlass::layout::TensorNDHWC> {
|
||||
static LayoutTypeID const kId = LayoutTypeID::kTensorNDHWC;
|
||||
};
|
||||
|
||||
template <> struct LayoutMap<cutlass::layout::TensorNCxHWx<32>> {
|
||||
static LayoutTypeID const kId = LayoutTypeID::kTensorNC32HW32;
|
||||
};
|
||||
|
||||
template <> struct LayoutMap<cutlass::layout::TensorNCxHWx<64>> {
|
||||
static LayoutTypeID const kId = LayoutTypeID::kTensorNC64HW64;
|
||||
};
|
||||
|
||||
template <> struct LayoutMap<cutlass::layout::TensorCxRSKx<32>> {
|
||||
static LayoutTypeID const kId = LayoutTypeID::kTensorC32RSK32;
|
||||
};
|
||||
|
||||
template <> struct LayoutMap<cutlass::layout::TensorCxRSKx<64>> {
|
||||
static LayoutTypeID const kId = LayoutTypeID::kTensorC64RSK64;
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename T> struct OpcodeClassMap;
|
||||
@@ -257,6 +274,43 @@ template <> struct ComplexTransformMap<cutlass::ComplexTransform::kConjugate> {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <cutlass::conv::Mode T> struct ConvModeMap;
|
||||
|
||||
template <> struct ConvModeMap<conv::Mode::kCrossCorrelation> {
|
||||
static ConvModeID const kId = ConvModeID::kCrossCorrelation;
|
||||
};
|
||||
|
||||
template <> struct ConvModeMap<conv::Mode::kConvolution> {
|
||||
static ConvModeID const kId = ConvModeID::kConvolution;
|
||||
};
|
||||
|
||||
|
||||
template <cutlass::conv::Operator T> struct ConvKindMap;
|
||||
|
||||
template <> struct ConvKindMap<conv::Operator::kFprop> {
|
||||
static ConvKind const kId = ConvKind::kFprop;
|
||||
};
|
||||
|
||||
template <> struct ConvKindMap<conv::Operator::kDgrad> {
|
||||
static ConvKind const kId = ConvKind::kDgrad;
|
||||
};
|
||||
|
||||
template <> struct ConvKindMap<conv::Operator::kWgrad> {
|
||||
static ConvKind const kId = ConvKind::kWgrad;
|
||||
};
|
||||
|
||||
|
||||
template <cutlass::conv::IteratorAlgorithm T> struct IteratorAlgorithmMap;
|
||||
|
||||
template <> struct IteratorAlgorithmMap<conv::IteratorAlgorithm::kAnalytic> {
|
||||
static IteratorAlgorithmID const kId = IteratorAlgorithmID::kAnalytic;
|
||||
};
|
||||
|
||||
template <> struct IteratorAlgorithmMap<conv::IteratorAlgorithm::kOptimized> {
|
||||
static IteratorAlgorithmID const kId = IteratorAlgorithmID::kOptimized;
|
||||
};
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Element, typename Layout>
|
||||
TensorDescription make_TensorDescription(int alignment = 1) {
|
||||
TensorDescription desc;
|
||||
|
||||
@@ -36,6 +36,11 @@ namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_reference_operations(Manifest &manifest);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Top-level initialization
|
||||
Status Manifest::initialize() {
|
||||
|
||||
@@ -46,6 +51,12 @@ Status Manifest::initialize() {
|
||||
// initialize procedurally generated cutlass op in manifest object
|
||||
initialize_all(*this);
|
||||
|
||||
// initialize manually instanced conv3d reference op in manifest object
|
||||
initialize_reference_operations(*this);
|
||||
|
||||
// initialize manually instanced reduction reference op in manifest object
|
||||
initialize_all_reduction_op(*this);
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
|
||||
@@ -76,6 +76,55 @@ void OperationTable::append(Manifest const &manifest) {
|
||||
}
|
||||
|
||||
|
||||
// insert all conv2d or conv3d operation into operation table
|
||||
if (desc.kind == OperationKind::kConv2d || desc.kind == OperationKind::kConv3d) {
|
||||
auto &conv_desc = static_cast<library::ConvDescription const &>(desc);
|
||||
|
||||
ConvFunctionalKey functional_key(
|
||||
conv_desc.provider,
|
||||
conv_desc.conv_kind,
|
||||
conv_desc.A.element,
|
||||
conv_desc.A.layout,
|
||||
conv_desc.B.element,
|
||||
conv_desc.B.layout,
|
||||
conv_desc.C.element,
|
||||
conv_desc.C.layout,
|
||||
conv_desc.tile_description.math_instruction.element_accumulator,
|
||||
conv_desc.element_epilogue
|
||||
);
|
||||
|
||||
Operation const *op = operation.get();
|
||||
|
||||
int cc = conv_desc.tile_description.minimum_compute_capability;
|
||||
|
||||
ConvPreferenceKey preference_key(cc, conv_desc.iterator_algorithm);
|
||||
|
||||
// insert conv operation to conv2d_operations or conv3d_operations map
|
||||
(desc.kind == OperationKind::kConv2d) ?
|
||||
conv2d_operations[functional_key][preference_key].push_back(op) :
|
||||
conv3d_operations[functional_key][preference_key].push_back(op);
|
||||
}
|
||||
|
||||
// insert all reduction operation into operation table
|
||||
if (desc.kind == OperationKind::kReduction) {
|
||||
auto &reduce_desc = static_cast<library::ReductionDescription const &>(desc);
|
||||
|
||||
ReductionFunctionalKey functional_key(
|
||||
reduce_desc.provider,
|
||||
reduce_desc.element_workspace,
|
||||
reduce_desc.tile_description.math_instruction.element_accumulator,
|
||||
reduce_desc.element_output,
|
||||
reduce_desc.element_epilogue,
|
||||
library::MathOperationID::kAdd,
|
||||
library::EpilogueKind::kLinearCombination
|
||||
);
|
||||
|
||||
Operation const *op = operation.get();
|
||||
|
||||
reduction_operations[functional_key] = op;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Initialize operations for reduction operation in CUTLASS Library.
|
||||
|
||||
*/
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// CUTLASS Reduction Instances //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void initialize_reduce_add_linear_combination_f32_f32_f16(Manifest &manifest);
|
||||
void initialize_reduce_add_linear_combination_f32_f32_f32(Manifest &manifest);
|
||||
void initialize_reduce_add_linear_combination_cf32_cf32_cf32(Manifest &manifest);
|
||||
|
||||
//
|
||||
// Entry point to construct operations
|
||||
//
|
||||
void initialize_all_reduction_op(Manifest &manifest) {
|
||||
|
||||
initialize_reduce_add_linear_combination_f32_f32_f16(manifest);
|
||||
initialize_reduce_add_linear_combination_f32_f32_f32(manifest);
|
||||
initialize_reduce_add_linear_combination_cf32_cf32_cf32(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,145 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for reduction operation in CUTLASS Library.
|
||||
*/
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "reduction_operation.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
// naming convention initialize_reduce_[ReductionOp]_[EpilogueOp]_[ElementWorkspace]_[ElementAccumulator]_[ElementOutput]
|
||||
|
||||
void initialize_reduce_add_linear_combination_f32_f32_f16(Manifest &manifest) {
|
||||
|
||||
using ElementWorkspace = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
128 / cutlass::sizeof_bits<ElementOutput>::value,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
using ReductionOp = cutlass::reduction::thread::ReduceAdd<
|
||||
ElementAccumulator,
|
||||
typename EpilogueOutputOp::ElementAccumulator,
|
||||
EpilogueOutputOp::kCount
|
||||
>;
|
||||
|
||||
using Operation_reduce_add_linear_combination_f32_f32_f16 = cutlass::reduction::device::ReduceSplitK<
|
||||
cutlass::reduction::kernel::ReduceSplitK<
|
||||
cutlass::MatrixShape<4, 32 * EpilogueOutputOp::kCount>,
|
||||
EpilogueOutputOp,
|
||||
ReductionOp
|
||||
>
|
||||
>;
|
||||
|
||||
manifest.append(new ReductionOperation<
|
||||
Operation_reduce_add_linear_combination_f32_f32_f16>(
|
||||
"reduce_add_linear_combination_f32_f32_f16"
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
void initialize_reduce_add_linear_combination_f32_f32_f32(Manifest &manifest) {
|
||||
|
||||
using ElementWorkspace = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementOutput = float;
|
||||
using ElementCompute = float;
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
128 / cutlass::sizeof_bits<ElementOutput>::value,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
using ReductionOp = cutlass::reduction::thread::ReduceAdd<
|
||||
ElementAccumulator,
|
||||
typename EpilogueOutputOp::ElementAccumulator,
|
||||
EpilogueOutputOp::kCount
|
||||
>;
|
||||
|
||||
using Operation_reduce_add_linear_combination_f32_f32_f32 = cutlass::reduction::device::ReduceSplitK<
|
||||
cutlass::reduction::kernel::ReduceSplitK<
|
||||
cutlass::MatrixShape<4, 32 * EpilogueOutputOp::kCount>,
|
||||
EpilogueOutputOp,
|
||||
ReductionOp
|
||||
>
|
||||
>;
|
||||
|
||||
manifest.append(new ReductionOperation<
|
||||
Operation_reduce_add_linear_combination_f32_f32_f32>(
|
||||
"reduce_add_linear_combination_f32_f32_f32"
|
||||
));
|
||||
}
|
||||
|
||||
void initialize_reduce_add_linear_combination_cf32_cf32_cf32(Manifest &manifest) {
|
||||
|
||||
using ElementWorkspace = cutlass::complex<float>;
|
||||
using ElementAccumulator = cutlass::complex<float>;
|
||||
using ElementOutput = cutlass::complex<float>;
|
||||
using ElementCompute = cutlass::complex<float>;
|
||||
|
||||
using EpilogueOutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
128 / cutlass::sizeof_bits<ElementOutput>::value,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
using ReductionOp = cutlass::reduction::thread::ReduceAdd<
|
||||
ElementAccumulator,
|
||||
typename EpilogueOutputOp::ElementAccumulator,
|
||||
EpilogueOutputOp::kCount
|
||||
>;
|
||||
|
||||
using Operation_reduce_add_linear_combination_cf32_cf32_cf32 = cutlass::reduction::device::ReduceSplitK<
|
||||
cutlass::reduction::kernel::ReduceSplitK<
|
||||
cutlass::MatrixShape<4, 32 * EpilogueOutputOp::kCount>,
|
||||
EpilogueOutputOp,
|
||||
ReductionOp
|
||||
>
|
||||
>;
|
||||
|
||||
manifest.append(new ReductionOperation<
|
||||
Operation_reduce_add_linear_combination_cf32_cf32_cf32>(
|
||||
"reduce_add_linear_combination_cf32_cf32_cf32"
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for reduction operation in CUTLASS Library.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
#include "cutlass/reduction/thread/reduction_operators.h"
|
||||
#include "cutlass/reduction/device/reduce_split_k.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "library_internal.h"
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Operator_>
|
||||
class ReductionOperation : public Operation {
|
||||
public:
|
||||
using Operator = Operator_;
|
||||
|
||||
using ElementWorkspace = typename Operator::ElementWorkspace;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementOutput = typename Operator::ElementOutput;
|
||||
|
||||
using ElementCompute = typename Operator::OutputOp::ElementCompute;
|
||||
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
|
||||
protected:
|
||||
|
||||
///
|
||||
ReductionDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
ReductionOperation(char const *name = "unknown_reduction") {
|
||||
|
||||
description_.name = name;
|
||||
description_.provider = Provider::kCUTLASS;
|
||||
description_.kind = OperationKind::kReduction;
|
||||
|
||||
description_.tile_description.threadblock_shape = make_Coord(Operator::Shape::kRow, Operator::Shape::kColumn, 1);
|
||||
|
||||
description_.tile_description.math_instruction.instruction_shape = make_Coord(1, 1, 1);
|
||||
description_.tile_description.math_instruction.element_accumulator = NumericTypeMap<ElementAccumulator>::kId;
|
||||
description_.tile_description.math_instruction.opcode_class = OpcodeClassID::kSimt;
|
||||
description_.tile_description.math_instruction.math_operation = MathOperationID::kAdd;
|
||||
|
||||
description_.tile_description.minimum_compute_capability = 50;
|
||||
description_.tile_description.maximum_compute_capability = 1024;
|
||||
|
||||
description_.element_workspace = NumericTypeMap<ElementWorkspace>::kId;
|
||||
description_.element_output = NumericTypeMap<ElementOutput>::kId;
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
}
|
||||
|
||||
/// Returns the description of the Reduction operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
|
||||
|
||||
protected:
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status construct_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
ReductionConfiguration const *configuration) {
|
||||
|
||||
operator_args.problem_size = configuration->problem_size;
|
||||
operator_args.partitions = configuration->partitions;
|
||||
operator_args.partition_stride = configuration->partition_stride;
|
||||
|
||||
operator_args.workspace = {nullptr, int(configuration->ldw)};
|
||||
operator_args.source = {nullptr, int(configuration->lds)};
|
||||
operator_args.destination = {nullptr, int(configuration->ldd)};
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status update_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
ReductionArguments const *arguments) {
|
||||
|
||||
if (arguments->pointer_mode == ScalarPointerMode::kHost) {
|
||||
typename Operator::OutputOp::Params params(
|
||||
*static_cast<ElementCompute const *>(arguments->alpha),
|
||||
*static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output = params;
|
||||
}
|
||||
else if (arguments->pointer_mode == ScalarPointerMode::kDevice){
|
||||
typename Operator::OutputOp::Params params(
|
||||
static_cast<ElementCompute const *>(arguments->alpha),
|
||||
static_cast<ElementCompute const *>(arguments->beta)
|
||||
);
|
||||
operator_args.output = params;
|
||||
}
|
||||
else {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
operator_args.workspace.reset(static_cast<ElementWorkspace *>(const_cast<void *>(arguments->workspace)));
|
||||
operator_args.source.reset(static_cast<ElementOutput *>(const_cast<void *>(arguments->source)));
|
||||
operator_args.destination.reset(static_cast<ElementOutput *>(const_cast<void *>(arguments->destination)));
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
/// Returns success if the operation can proceed
|
||||
virtual Status can_implement(
|
||||
void const *configuration_ptr,
|
||||
void const *arguments_ptr) const {
|
||||
|
||||
ReductionConfiguration const *configuration =
|
||||
static_cast<ReductionConfiguration const *>(configuration_ptr);
|
||||
|
||||
ReductionArguments const *arguments =
|
||||
static_cast<ReductionArguments const *>(arguments_ptr);
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(args, configuration);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
status = update_arguments_(args, arguments);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
return Operator::can_implement(args);
|
||||
}
|
||||
|
||||
/// Gets the host-side workspace
|
||||
virtual uint64_t get_host_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
return sizeof(Operator);
|
||||
}
|
||||
|
||||
/// Gets the device-side workspace
|
||||
virtual uint64_t get_device_workspace_size(
|
||||
void const *configuration_ptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<ReductionConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return Operator::get_workspace_size(args);
|
||||
}
|
||||
|
||||
/// Initializes the workspace
|
||||
virtual Status initialize(
|
||||
void const *configuration_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = construct_arguments_(
|
||||
args,
|
||||
static_cast<ReductionConfiguration const *>(configuration_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = new (host_workspace) Operator;
|
||||
//std::cout << "initialize library::Reduction" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->initialize(args, device_workspace, stream);
|
||||
}
|
||||
|
||||
/// Runs the kernel
|
||||
virtual Status run(
|
||||
void const *arguments_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
OperatorArguments args;
|
||||
|
||||
Status status = update_arguments_(
|
||||
args,
|
||||
static_cast<ReductionArguments const *>(arguments_ptr));
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = static_cast<Operator *>(host_workspace);
|
||||
|
||||
status = op->update(args, device_workspace);
|
||||
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
//std::cout << "run library::Reduction" << std::endl;
|
||||
//print_operator_args(args);
|
||||
return op->run(stream);
|
||||
}
|
||||
|
||||
/// Call print_operator_args from the Reduction::initialize()
|
||||
// to dump arguments passed on to cutlass operator for debugging
|
||||
void print_operator_args(OperatorArguments &operator_args) const {
|
||||
std::cout << "Reduction::OperatorArguments" << std::endl
|
||||
<< " problem_size: "
|
||||
<< operator_args.problem_size << std::endl
|
||||
<< " partitions: "
|
||||
<< operator_args.partitions << std::endl
|
||||
<< " partition_stride: "
|
||||
<< operator_args.partition_stride << std::endl
|
||||
<< " epilouge (alpha, beta): "
|
||||
<< operator_args.output.alpha << ", "
|
||||
<< operator_args.output.beta << std::endl
|
||||
<< " workspace (ptr, stride): "
|
||||
<< operator_args.workspace.data() << ", "
|
||||
<< operator_args.workspace.stride(0) << std::endl
|
||||
<< " source (ptr, stride): "
|
||||
<< operator_args.source.data() << ", "
|
||||
<< operator_args.source.stride(0) << std::endl
|
||||
<< " destination (ptr, stride): "
|
||||
<< operator_args.destination.data() << ", "
|
||||
<< operator_args.destination.stride(0) << std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,223 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief
|
||||
|
||||
*/
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "conv_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_conv2d_reference_operations(Manifest &manifest) {
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNHWC,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNHWC,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNHWC,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
2,
|
||||
cutlass::complex<float>, cutlass::layout::TensorNHWC,
|
||||
cutlass::complex<float>, cutlass::layout::TensorNHWC,
|
||||
cutlass::complex<float>, cutlass::layout::TensorNHWC,
|
||||
cutlass::complex<float>,
|
||||
cutlass::complex<float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
int32_t, cutlass::layout::TensorNHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<int8_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<uint8_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
int32_t, cutlass::layout::TensorNHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
uint8_t, cutlass::layout::TensorNHWC,
|
||||
int8_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<int8_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNHWC,
|
||||
int32_t, cutlass::layout::TensorNHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<cutlass::int4b_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNHWC,
|
||||
int32_t, cutlass::layout::TensorNHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
2,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<cutlass::uint4b_t, float>
|
||||
>(manifest);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief
|
||||
*/
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "conv_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_conv3d_reference_operations(Manifest &manifest) {
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t,
|
||||
cutlass::half_t
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::half_t, cutlass::layout::TensorNDHWC,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::bfloat16_t, cutlass::layout::TensorNDHWC,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::tfloat32_t, cutlass::layout::TensorNDHWC,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_all<
|
||||
3,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
float
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<int8_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
uint8_t, cutlass::layout::TensorNDHWC,
|
||||
uint8_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
uint8_t, cutlass::layout::TensorNDHWC,
|
||||
uint8_t, cutlass::layout::TensorNDHWC,
|
||||
int8_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<int8_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::int4b_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<cutlass::int4b_t, float>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t, cutlass::layout::TensorNDHWC,
|
||||
int32_t,
|
||||
int32_t,
|
||||
NumericConverterClamp<int32_t, int32_t>
|
||||
>(manifest);
|
||||
|
||||
make_conv_fprop<
|
||||
3,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNDHWC,
|
||||
cutlass::uint4b_t, cutlass::layout::TensorNDHWC,
|
||||
float,
|
||||
int32_t,
|
||||
NumericConverterClamp<cutlass::uint4b_t, float>
|
||||
>(manifest);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,607 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for all CONV operation kinds in CUTLASS Library
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <cstring>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "library_internal.h"
|
||||
|
||||
#include "cutlass/util/reference/host/convolution.h"
|
||||
#include "cutlass/util/reference/device/convolution.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <
|
||||
Provider kProvider,
|
||||
conv::Operator ConvolutionalOperator,
|
||||
int ConvDim,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
struct ConvReferenceDispatcher;
|
||||
|
||||
/// Dispatcher for Conv2d (partially specialied for kConvDim == 2)
|
||||
template <
|
||||
Provider kProvider,
|
||||
conv::Operator kConvolutionalOperator,
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator,
|
||||
typename ConvertOp,
|
||||
typename InnerProductOp
|
||||
>
|
||||
struct ConvReferenceDispatcher<
|
||||
kProvider,
|
||||
kConvolutionalOperator,
|
||||
2,
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp> {
|
||||
|
||||
static Status dispatch(
|
||||
void const *configuration,
|
||||
ElementA *ptr_A,
|
||||
ElementB *ptr_B,
|
||||
ElementC *ptr_C,
|
||||
ElementC *ptr_D,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta,
|
||||
cudaStream_t stream = nullptr
|
||||
) {
|
||||
|
||||
Conv2dConfiguration const &config =
|
||||
*static_cast<Conv2dConfiguration const *>(configuration);
|
||||
|
||||
ConvKind const conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
|
||||
|
||||
if (kProvider == Provider::kReferenceHost) {
|
||||
|
||||
cutlass::reference::host::Conv2d<
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC ,
|
||||
LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp
|
||||
>(
|
||||
kConvolutionalOperator,
|
||||
config.problem_size,
|
||||
{ptr_A, config.layout_a(conv_kind)},
|
||||
{ptr_B, config.layout_b(conv_kind)},
|
||||
{ptr_C, config.layout_c(conv_kind)},
|
||||
{ptr_D, config.layout_c(conv_kind)},
|
||||
alpha,
|
||||
beta
|
||||
);
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else if (kProvider == Provider::kReferenceDevice) {
|
||||
return cutlass::reference::device::Conv2d<
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp
|
||||
>(
|
||||
kConvolutionalOperator,
|
||||
config.problem_size,
|
||||
{ptr_A, config.layout_a(conv_kind)},
|
||||
{ptr_B, config.layout_b(conv_kind)},
|
||||
{ptr_C, config.layout_c(conv_kind)},
|
||||
{ptr_D, config.layout_c(conv_kind)},
|
||||
alpha,
|
||||
beta,
|
||||
stream
|
||||
);
|
||||
}
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
};
|
||||
|
||||
/// Dispatcher for Conv3d (partially specialized for kConvDim == 3)
|
||||
template <
|
||||
Provider kProvider,
|
||||
conv::Operator kConvolutionalOperator,
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator,
|
||||
typename ConvertOp,
|
||||
typename InnerProductOp
|
||||
>
|
||||
struct ConvReferenceDispatcher<
|
||||
kProvider,
|
||||
kConvolutionalOperator,
|
||||
3,
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp> {
|
||||
|
||||
static Status dispatch(
|
||||
void const *configuration,
|
||||
ElementA *ptr_A,
|
||||
ElementB *ptr_B,
|
||||
ElementC *ptr_C,
|
||||
ElementC *ptr_D,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta,
|
||||
cudaStream_t stream = nullptr
|
||||
) {
|
||||
|
||||
Conv3dConfiguration const &config =
|
||||
*static_cast<Conv3dConfiguration const *>(configuration);
|
||||
|
||||
ConvKind const conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
|
||||
|
||||
if (kProvider == Provider::kReferenceHost) {
|
||||
cutlass::reference::host::Conv3d<
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC ,
|
||||
LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp
|
||||
>(
|
||||
kConvolutionalOperator,
|
||||
config.problem_size,
|
||||
{ptr_A, config.layout_a(conv_kind)},
|
||||
{ptr_B, config.layout_b(conv_kind)},
|
||||
{ptr_C, config.layout_c(conv_kind)},
|
||||
{ptr_D, config.layout_c(conv_kind)},
|
||||
alpha,
|
||||
beta
|
||||
);
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else if (kProvider == Provider::kReferenceDevice) {
|
||||
return cutlass::reference::device::Conv3d<
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp
|
||||
>(
|
||||
kConvolutionalOperator,
|
||||
config.problem_size,
|
||||
{ptr_A, config.layout_a(conv_kind)},
|
||||
{ptr_B, config.layout_b(conv_kind)},
|
||||
{ptr_C, config.layout_c(conv_kind)},
|
||||
{ptr_D, config.layout_c(conv_kind)},
|
||||
alpha,
|
||||
beta,
|
||||
stream
|
||||
);
|
||||
}
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
Provider Provider_,
|
||||
conv::Operator ConvolutionalOperator,
|
||||
int ConvDim,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
class ConvReferenceOperation : public Operation {
|
||||
public:
|
||||
static Provider const kProvider = Provider_;
|
||||
static conv::Operator const kConvolutionalOperator = ConvolutionalOperator;
|
||||
static int const kConvDim = ConvDim;
|
||||
|
||||
using ElementA = ElementA_;
|
||||
using LayoutA = LayoutA_;
|
||||
using ElementB = ElementB_;
|
||||
using LayoutB = LayoutB_;
|
||||
using ElementC = ElementC_;
|
||||
using LayoutC = LayoutC_;
|
||||
using ElementCompute = ElementCompute_;
|
||||
using ElementAccumulator = ElementAccumulator_;
|
||||
using ConvertOp = ConvertOp_;
|
||||
using InnerProductOp = InnerProductOp_;
|
||||
|
||||
protected:
|
||||
|
||||
/// Storage for the name string
|
||||
std::string name_;
|
||||
|
||||
///
|
||||
ConvDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
ConvReferenceOperation() {
|
||||
|
||||
// Basic information
|
||||
description_.provider = kProvider;
|
||||
description_.kind = (kConvDim == 2 ? OperationKind::kConv2d : OperationKind::kConv3d);
|
||||
description_.conv_kind = ConvKindMap<kConvolutionalOperator>::kId;
|
||||
description_.conv_dim = kConvDim;
|
||||
|
||||
// Tensor description
|
||||
description_.A = make_TensorDescription<ElementA, LayoutA>();
|
||||
description_.B = make_TensorDescription<ElementB, LayoutB>();
|
||||
description_.C = make_TensorDescription<ElementC, LayoutC>();
|
||||
|
||||
// Epilogue compute and accumulator type description
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
|
||||
// Iterator algorithm for convolution reference
|
||||
description_.iterator_algorithm = IteratorAlgorithmID::kNone;
|
||||
|
||||
// Compute capability for convolution reference
|
||||
description_.tile_description.minimum_compute_capability =
|
||||
(kProvider == Provider::kReferenceDevice ? 50 : 0);
|
||||
|
||||
description_.tile_description.maximum_compute_capability = 1024;
|
||||
|
||||
// Procedural name
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "conv" << kConvDim << "d_" << to_string(description_.conv_kind)
|
||||
<< "_reference_" << to_string(description_.provider)
|
||||
<< "_" << to_string(description_.A.element) << to_string(description_.A.layout)
|
||||
<< "_" << to_string(description_.B.element) << to_string(description_.B.layout)
|
||||
<< "_" << to_string(description_.C.element) << to_string(description_.C.layout)
|
||||
<< "_" << to_string(description_.tile_description.math_instruction.element_accumulator);
|
||||
|
||||
name_ = ss.str();
|
||||
|
||||
description_.name = name_.c_str();
|
||||
|
||||
// Epilogue compute and accumulator type description
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
|
||||
virtual Status can_implement(
|
||||
void const *configuration,
|
||||
void const *arguments) const {
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
virtual uint64_t get_host_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
switch (kConvDim) {
|
||||
case 2:
|
||||
return sizeof(Conv2dConfiguration);
|
||||
case 3:
|
||||
return sizeof(Conv3dConfiguration);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual uint64_t get_device_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual Status initialize(
|
||||
void const *configuration,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
std::memcpy(host_workspace, configuration, get_host_workspace_size(configuration));
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
virtual Status run(
|
||||
void const *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
|
||||
ConvArguments const &args = *static_cast<ConvArguments const *>(arguments);
|
||||
|
||||
ElementCompute alpha;
|
||||
ElementCompute beta;
|
||||
|
||||
alpha = *static_cast<ElementCompute const *>(args.alpha);
|
||||
beta = *static_cast<ElementCompute const *>(args.beta);
|
||||
|
||||
// TODO - respect pointer mode
|
||||
|
||||
// Invoke 2D or 3D convolution
|
||||
return detail::ConvReferenceDispatcher<
|
||||
kProvider,
|
||||
kConvolutionalOperator,
|
||||
kConvDim,
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp,
|
||||
InnerProductOp
|
||||
>::dispatch(
|
||||
host_workspace,
|
||||
static_cast<ElementA *>(const_cast<void *>(args.A)),
|
||||
static_cast<ElementB *>(const_cast<void *>(args.B)),
|
||||
static_cast<ElementC *>(const_cast<void *>(args.C)),
|
||||
static_cast<ElementC *>(args.D),
|
||||
alpha,
|
||||
beta,
|
||||
stream
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Constructs Fprop reference operators.
|
||||
template <
|
||||
int kConvDim,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
void make_conv_fprop(Manifest &manifest) {
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
conv::Operator::kFprop,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceDevice,
|
||||
conv::Operator::kFprop,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
}
|
||||
|
||||
/// Constructs Dgrad and Wgrad reference operators.
|
||||
template <
|
||||
int kConvDim,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
void make_conv_backwards(Manifest &manifest) {
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
conv::Operator::kDgrad,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceDevice,
|
||||
conv::Operator::kDgrad,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
conv::Operator::kWgrad,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
|
||||
manifest.append(new ConvReferenceOperation<
|
||||
Provider::kReferenceDevice,
|
||||
conv::Operator::kWgrad,
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
}
|
||||
|
||||
/// Six operators for the price of one.
|
||||
template <
|
||||
int kConvDim,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
void make_conv_all(Manifest &manifest) {
|
||||
|
||||
make_conv_fprop<
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>(manifest);
|
||||
|
||||
make_conv_backwards<
|
||||
kConvDim,
|
||||
ElementA_, LayoutA_,
|
||||
ElementB_, LayoutB_,
|
||||
ElementC_, LayoutC_,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>(manifest);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -37,10 +37,14 @@ namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
void initialize_gemm_reference_operations(Manifest &manifest);
|
||||
void initialize_conv2d_reference_operations(Manifest &manifest);
|
||||
void initialize_conv3d_reference_operations(Manifest &manifest);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_reference_operations(Manifest &manifest) {
|
||||
initialize_conv2d_reference_operations(manifest);
|
||||
initialize_conv3d_reference_operations(manifest);
|
||||
initialize_gemm_reference_operations(manifest);
|
||||
}
|
||||
|
||||
|
||||
+168
-8
@@ -50,6 +50,7 @@ Provider_enumerants[] = {
|
||||
{"host", "reference_host", Provider::kReferenceHost},
|
||||
{"device", "reference_device", Provider::kReferenceDevice},
|
||||
{"cublas", "cuBLAS", Provider::kCUBLAS},
|
||||
{"cudnn", "cuDNN", Provider::kCUDNN},
|
||||
};
|
||||
|
||||
/// Converts a Provider enumerant to a string
|
||||
@@ -128,6 +129,9 @@ static struct {
|
||||
OperationKind_enumerants[] = {
|
||||
{"eq_gemm", "EqGemm", OperationKind::kEqGemm},
|
||||
{"gemm", "Gemm", OperationKind::kGemm},
|
||||
{"conv2d", "Conv2d", OperationKind::kConv2d},
|
||||
{"conv3d", "Conv3d", OperationKind::kConv3d},
|
||||
{"spgemm", "SparseGemm", OperationKind::kSparseGemm},
|
||||
};
|
||||
|
||||
/// Converts a Status enumerant to a string
|
||||
@@ -445,6 +449,10 @@ layout_aliases[] = {
|
||||
{LayoutTypeID::kTensorNCDHW, "ncdhw"},
|
||||
{LayoutTypeID::kTensorNHWC, "nhwc"},
|
||||
{LayoutTypeID::kTensorNDHWC, "ndhwc"},
|
||||
{LayoutTypeID::kTensorNC32HW32, "nc32hw32"},
|
||||
{LayoutTypeID::kTensorNC64HW64, "nc64hw64"},
|
||||
{LayoutTypeID::kTensorC32RSK32, "c32rsk32"},
|
||||
{LayoutTypeID::kTensorC64RSK64, "c64rsk64"},
|
||||
|
||||
{LayoutTypeID::kUnknown, "*"},
|
||||
{LayoutTypeID::kInvalid, nullptr}
|
||||
@@ -474,22 +482,46 @@ LayoutTypeID from_string<LayoutTypeID>(std::string const &str) {
|
||||
/// Gets stride rank for the layout_id (static function)
|
||||
int get_layout_stride_rank(LayoutTypeID layout_id) {
|
||||
switch (layout_id) {
|
||||
case LayoutTypeID::kColumnMajor: return cutlass::layout::ColumnMajor::kStrideRank;
|
||||
case LayoutTypeID::kRowMajor: return cutlass::layout::RowMajor::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajor:
|
||||
return cutlass::layout::ColumnMajor::kStrideRank;
|
||||
case LayoutTypeID::kRowMajor:
|
||||
return cutlass::layout::RowMajor::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajorInterleavedK2:
|
||||
return cutlass::layout::ColumnMajorInterleaved<2>::kStrideRank;
|
||||
case LayoutTypeID::kRowMajorInterleavedK2:
|
||||
return cutlass::layout::RowMajorInterleaved<2>::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajorInterleavedK4:
|
||||
return cutlass::layout::ColumnMajorInterleaved<4>::kStrideRank;
|
||||
case LayoutTypeID::kRowMajorInterleavedK4:
|
||||
return cutlass::layout::RowMajorInterleaved<4>::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajorInterleavedK16:
|
||||
return cutlass::layout::ColumnMajorInterleaved<16>::kStrideRank;
|
||||
case LayoutTypeID::kRowMajorInterleavedK16:
|
||||
return cutlass::layout::RowMajorInterleaved<16>::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajorInterleavedK32:
|
||||
return cutlass::layout::ColumnMajorInterleaved<32>::kStrideRank;
|
||||
case LayoutTypeID::kRowMajorInterleavedK32:
|
||||
return cutlass::layout::RowMajorInterleaved<32>::kStrideRank;
|
||||
case LayoutTypeID::kColumnMajorInterleavedK64:
|
||||
case LayoutTypeID::kRowMajorInterleavedK64: return 1;
|
||||
return cutlass::layout::ColumnMajorInterleaved<64>::kStrideRank;
|
||||
case LayoutTypeID::kRowMajorInterleavedK64:
|
||||
return cutlass::layout::RowMajorInterleaved<64>::kStrideRank;
|
||||
case LayoutTypeID::kTensorNCHW:
|
||||
case LayoutTypeID::kTensorNHWC: return 3;
|
||||
case LayoutTypeID::kTensorNDHWC: return 4;
|
||||
default : throw std::runtime_error("Unsupported LayoutTypeID in LayoutType::get_stride_rank");
|
||||
return cutlass::layout::TensorNCHW::kStrideRank;
|
||||
case LayoutTypeID::kTensorNHWC:
|
||||
return cutlass::layout::TensorNHWC::kStrideRank;
|
||||
case LayoutTypeID::kTensorNDHWC:
|
||||
return cutlass::layout::TensorNDHWC::kStrideRank;
|
||||
case LayoutTypeID::kTensorNC32HW32:
|
||||
return cutlass::layout::TensorNCxHWx<32>::kStrideRank;
|
||||
case LayoutTypeID::kTensorNC64HW64:
|
||||
return cutlass::layout::TensorNCxHWx<64>::kStrideRank;
|
||||
case LayoutTypeID::kTensorC32RSK32:
|
||||
return cutlass::layout::TensorCxRSKx<32>::kStrideRank;
|
||||
case LayoutTypeID::kTensorC64RSK64:
|
||||
return cutlass::layout::TensorCxRSKx<64>::kStrideRank;
|
||||
default:
|
||||
throw std::runtime_error("Unsupported LayoutTypeID in LayoutType::get_stride_rank");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -624,6 +656,136 @@ SplitKMode from_string<SplitKMode>(std::string const &str) {
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
static struct {
|
||||
char const *text;
|
||||
char const *pretty;
|
||||
ConvModeID enumerant;
|
||||
}
|
||||
ConvModeID_enumerants[] = {
|
||||
{"cross", "<cross>", ConvModeID::kCrossCorrelation},
|
||||
{"conv", "<conv>", ConvModeID::kConvolution},
|
||||
};
|
||||
|
||||
/// Converts a ConvModeID enumerant to a string
|
||||
char const *to_string(ConvModeID type, bool pretty) {
|
||||
|
||||
for (auto const & possible : ConvModeID_enumerants) {
|
||||
if (type == possible.enumerant) {
|
||||
if (pretty) {
|
||||
return possible.pretty;
|
||||
}
|
||||
else {
|
||||
return possible.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pretty ? "Invalid" : "invalid";
|
||||
}
|
||||
|
||||
/// Converts a ConvModeID enumerant from a string
|
||||
template <>
|
||||
ConvModeID from_string<ConvModeID>(std::string const &str) {
|
||||
|
||||
for (auto const & possible : ConvModeID_enumerants) {
|
||||
if ((str.compare(possible.text) == 0) ||
|
||||
(str.compare(possible.pretty) == 0)) {
|
||||
return possible.enumerant;
|
||||
}
|
||||
}
|
||||
|
||||
return ConvModeID::kInvalid;
|
||||
}
|
||||
|
||||
|
||||
static struct {
|
||||
char const *text;
|
||||
char const *pretty;
|
||||
IteratorAlgorithmID enumerant;
|
||||
}
|
||||
IteratorAlgorithmID_enumerants[] = {
|
||||
{"none", "<none>", IteratorAlgorithmID::kNone},
|
||||
{"analytic", "<analytic>", IteratorAlgorithmID::kAnalytic},
|
||||
{"optimized", "<optimized>", IteratorAlgorithmID::kOptimized},
|
||||
};
|
||||
|
||||
/// Converts a ConvModeID enumerant to a string
|
||||
char const *to_string(IteratorAlgorithmID type, bool pretty) {
|
||||
|
||||
for (auto const & possible : IteratorAlgorithmID_enumerants) {
|
||||
if (type == possible.enumerant) {
|
||||
if (pretty) {
|
||||
return possible.pretty;
|
||||
}
|
||||
else {
|
||||
return possible.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pretty ? "Invalid" : "invalid";
|
||||
}
|
||||
|
||||
/// Converts a ConvModeID enumerant from a string
|
||||
template <>
|
||||
IteratorAlgorithmID from_string<IteratorAlgorithmID>(std::string const &str) {
|
||||
|
||||
for (auto const & possible : IteratorAlgorithmID_enumerants) {
|
||||
if ((str.compare(possible.text) == 0) ||
|
||||
(str.compare(possible.pretty) == 0)) {
|
||||
return possible.enumerant;
|
||||
}
|
||||
}
|
||||
|
||||
return IteratorAlgorithmID::kInvalid;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static struct {
|
||||
char const *text;
|
||||
char const *pretty;
|
||||
ConvKind enumerant;
|
||||
}
|
||||
ConvKind_enumerants[] = {
|
||||
{"unknown", "<unkown>", ConvKind::kUnknown},
|
||||
{"fprop", "<fprop>", ConvKind::kFprop},
|
||||
{"dgrad", "<dgrad>", ConvKind::kDgrad},
|
||||
{"wgrad", "<wgrad>", ConvKind::kWgrad},
|
||||
};
|
||||
|
||||
/// Converts a ConvKind enumerant to a string
|
||||
char const *to_string(ConvKind type, bool pretty) {
|
||||
|
||||
for (auto const & possible : ConvKind_enumerants) {
|
||||
if (type == possible.enumerant) {
|
||||
if (pretty) {
|
||||
return possible.pretty;
|
||||
}
|
||||
else {
|
||||
return possible.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pretty ? "Invalid" : "invalid";
|
||||
}
|
||||
|
||||
|
||||
/// Converts a ConvKind enumerant from a string
|
||||
template <>
|
||||
ConvKind from_string<ConvKind>(std::string const &str) {
|
||||
|
||||
for (auto const & possible : ConvKind_enumerants) {
|
||||
if ((str.compare(possible.text) == 0) ||
|
||||
(str.compare(possible.pretty) == 0)) {
|
||||
return possible.enumerant;
|
||||
}
|
||||
}
|
||||
|
||||
return ConvKind::kInvalid;
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Lexical cast a string to a byte array. Returns true if cast is successful or false if invalid.
|
||||
bool lexical_cast(std::vector<uint8_t> &bytes, NumericTypeID type, std::string const &str) {
|
||||
int size_bytes = sizeof_bits(type) / 8;
|
||||
@@ -1224,5 +1386,3 @@ bool cast_from_double(std::vector<uint8_t> &bytes, NumericTypeID type, double sr
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
@@ -34,9 +34,12 @@ set(CUTLASS_TOOLS_PROFILER_SOURCES
|
||||
src/device_allocation.cu
|
||||
src/device_context.cu
|
||||
src/cublas_helpers.cpp
|
||||
src/cudnn_helpers.cpp
|
||||
src/problem_space.cpp
|
||||
src/operation_profiler.cu
|
||||
src/gemm_operation_profiler.cu
|
||||
src/conv2d_operation_profiler.cu
|
||||
src/conv3d_operation_profiler.cu
|
||||
src/sparse_gemm_operation_profiler.cu
|
||||
)
|
||||
|
||||
@@ -58,7 +61,7 @@ set_target_properties(cutlass_profiler PROPERTIES EXPORT_NAME profiler)
|
||||
target_include_directories(
|
||||
cutlass_profiler
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src # Source directory
|
||||
${CMAKE_CURRENT_LIST_DIR}/src
|
||||
)
|
||||
|
||||
#
|
||||
@@ -71,6 +74,7 @@ target_link_libraries(
|
||||
cutlass_lib
|
||||
cutlass_tools_util_includes
|
||||
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:nvidia::cublas>
|
||||
$<$<BOOL:${CUTLASS_ENABLE_CUDNN}>:nvidia::cudnn>
|
||||
cudart
|
||||
)
|
||||
|
||||
@@ -79,3 +83,16 @@ install(
|
||||
EXPORT NvidiaCutlass
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
)
|
||||
|
||||
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM --operation=Gemm --providers=cutlass --verification-providers=cublas,device --junit-output=test_cutlass_profiler_gemm)
|
||||
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV2D --operation=Conv2d --providers=cutlass --verification-providers=cudnn,device --junit-output=test_cutlass_profiler_conv2d)
|
||||
set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV3D --operation=Conv3d --providers=cutlass --verification-providers=cudnn,device,host --junit-output=test_cutlass_profiler_conv3d)
|
||||
cutlass_add_executable_tests(
|
||||
test_profiler cutlass_profiler
|
||||
DEPENDEES test_all
|
||||
TEST_COMMAND_OPTIONS
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV2D
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV3D
|
||||
DISABLE_EXECUTABLE_INSTALL_RULE
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,431 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for convolution
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/handle.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/singleton.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#include "reduction_operation_profiler.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class Conv2dOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct Conv2dProblem {
|
||||
|
||||
int64_t n, h, w, c, p, q, k, r, s;
|
||||
int64_t pad_h, pad_w;
|
||||
int64_t stride_h, stride_w;
|
||||
int64_t dilation_h, dilation_w;
|
||||
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
|
||||
library::SplitKMode split_k_mode;
|
||||
int64_t split_k_slices;
|
||||
|
||||
library::ConvModeID conv_mode;
|
||||
|
||||
library::Provider eq_gemm_provider;
|
||||
|
||||
// convolution with parallel interleaved reduction
|
||||
// convolution epilogue (alpha, beta) = (1.0, 0.0)
|
||||
// reduction epilogue (alpha, beta) = (Conv2dProblem::alpha, Conv2dProblem::beta)
|
||||
std::vector<uint8_t> alpha_one;
|
||||
std::vector<uint8_t> beta_zero;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
void set_default_output_size() {
|
||||
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
|
||||
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
|
||||
}
|
||||
|
||||
// Returns equivalent gemm problem size for convolution
|
||||
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * p * q), int(k), int(r * s * c));
|
||||
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * h * w), int(c), int(k * r * s));
|
||||
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(r * s * c), int(n * p * q));
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor A
|
||||
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(p), int(q), int(k)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor B
|
||||
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(k), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(k), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(h), int(w), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor C
|
||||
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(k), int(r), int(s), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix A
|
||||
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix B
|
||||
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix C
|
||||
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
// Gemm operator assumes column-major output
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix A
|
||||
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix B
|
||||
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix C
|
||||
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct Conv2dWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments for convolution operator
|
||||
library::Conv2dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
|
||||
/// Number of copies of the problem workspace which are visited sequentially during
|
||||
/// profiling to avoid camping in the last level cache.
|
||||
int problem_count;
|
||||
|
||||
/// Buffer used for the cutlass conv2d operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
/// Library configuration and arguments for reduction operator
|
||||
library::ReductionConfiguration reduction_configuration;
|
||||
library::ReductionArguments reduction_arguments;
|
||||
|
||||
/// Buffer used for the cutlass reduction operations' host workspace
|
||||
std::vector<uint8_t> reduction_host_workspace;
|
||||
|
||||
/// Host data buffers for host reference operation
|
||||
/// host buffer for tensor
|
||||
std::vector<uint8_t> host_tensor_a;
|
||||
|
||||
/// host buffer for tensor b
|
||||
std::vector<uint8_t> host_tensor_b;
|
||||
|
||||
/// host buffer for tensor c
|
||||
std::vector<uint8_t> host_tensor_c;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Conv2dWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
|
||||
// Returns stride vector for tensor A
|
||||
std::vector<int> stride_a(library::ConvKind const &conv_kind) {
|
||||
return {
|
||||
configuration.layout_a(conv_kind).stride()[0],
|
||||
configuration.layout_a(conv_kind).stride()[1],
|
||||
configuration.layout_a(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor B
|
||||
std::vector<int> stride_b(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_b(conv_kind).stride()[0],
|
||||
configuration.layout_b(conv_kind).stride()[1],
|
||||
configuration.layout_b(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor C
|
||||
std::vector<int> stride_c(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_c(conv_kind).stride()[0],
|
||||
configuration.layout_c(conv_kind).stride()[1],
|
||||
configuration.layout_c(conv_kind).stride()[2]
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// CONV problem obtained from problem space
|
||||
Conv2dProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
Conv2dWorkspace conv_workspace_;
|
||||
|
||||
/// CUTLASS parallel reduction operation to follow this* conv2d operation
|
||||
library::Operation const *reduction_op_;
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
Conv2dOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~Conv2dOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
/// Method to profile an initialized CUTLASS operation
|
||||
virtual Status profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
|
||||
/// Initialize reduction problem dimenstions and library::Operation
|
||||
bool initialize_reduction_configuration_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::ConvDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against host reference
|
||||
bool verify_with_host_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against device reference
|
||||
bool verify_with_device_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
/// Verifies CUTLASS against cudnn reference
|
||||
bool verify_with_cudnn_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,441 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for convolution
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/handle.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/singleton.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#include "reduction_operation_profiler.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class Conv3dOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
/// Problem structure obtained from problem space
|
||||
struct Conv3dProblem {
|
||||
|
||||
int64_t n, d, h, w, c, z, p, q, k, t, r, s;
|
||||
int64_t pad_d, pad_h, pad_w;
|
||||
int64_t stride_d, stride_h, stride_w;
|
||||
int64_t dilation_d, dilation_h, dilation_w;
|
||||
|
||||
std::vector<uint8_t> alpha;
|
||||
std::vector<uint8_t> beta;
|
||||
|
||||
library::SplitKMode split_k_mode;
|
||||
int64_t split_k_slices;
|
||||
|
||||
library::ConvModeID conv_mode;
|
||||
|
||||
library::Provider eq_gemm_provider;
|
||||
|
||||
// convolution with parallel interleaved reduction
|
||||
// convolution epilogue (alpha, beta) = (1.0, 0.0)
|
||||
// reduction epilogue (alpha, beta) = (Conv3dProblem::alpha, Conv3dProblem::beta)
|
||||
std::vector<uint8_t> alpha_one;
|
||||
std::vector<uint8_t> beta_zero;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Total number of bytes loaded
|
||||
int64_t bytes(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Total number of flops computed
|
||||
int64_t flops(library::ConvDescription const &operation_desc) const;
|
||||
|
||||
/// Infers output size from theinput size, padding, stride, and dilation
|
||||
void set_default_output_size() {
|
||||
z = ((d + pad_d - t * dilation_d) / stride_d) + 1;
|
||||
p = ((h + pad_h - r * dilation_h) / stride_h) + 1;
|
||||
q = ((w + pad_w - s * dilation_w) / stride_w) + 1;
|
||||
}
|
||||
|
||||
// Returns equivalent gemm problem size for convolution
|
||||
cutlass::gemm::GemmCoord eq_gemm_size(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return cutlass::gemm::GemmCoord(int(n * z * p * q), int(k), int(t * r * s * c));
|
||||
case library::ConvKind::kDgrad: return cutlass::gemm::GemmCoord(int(n * d * h * w), int(c), int(t * r * s * k));
|
||||
case library::ConvKind::kWgrad: return cutlass::gemm::GemmCoord(int(k), int(t * r * s * c), int(n * z * p * q));
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor A
|
||||
std::vector<int> extent_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor B
|
||||
std::vector<int> extent_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kDgrad: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns extent for tensor C
|
||||
std::vector<int> extent_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return {int(n), int(z), int(p), int(q), int(k)};
|
||||
case library::ConvKind::kDgrad: return {int(n), int(d), int(h), int(w), int(c)};
|
||||
case library::ConvKind::kWgrad: return {int(k), int(t), int(r), int(s), int(c)};
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix A
|
||||
library::LayoutTypeID eq_gemm_layout_a(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kRowMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix B
|
||||
library::LayoutTypeID eq_gemm_layout_b(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return library::LayoutTypeID::kColumnMajor; // TN Gemm
|
||||
case library::ConvKind::kDgrad: return library::LayoutTypeID::kRowMajor; // TT Gemm
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kRowMajor; // NT Gemm
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns layout for equivalent gemm matrix C
|
||||
library::LayoutTypeID eq_gemm_layout_c(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
// Gemm operator assumes column-major output
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return library::LayoutTypeID::kColumnMajor;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix A
|
||||
int64_t eq_gemm_lda(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix B
|
||||
int64_t eq_gemm_ldb(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop: return eq_gemm_size(conv_kind).k();
|
||||
case library::ConvKind::kDgrad: return eq_gemm_size(conv_kind).n();
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).n();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns leading dimenstion for equivalent gemm matrix C
|
||||
int64_t eq_gemm_ldc(library::ConvKind const &conv_kind) const {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
case library::ConvKind::kDgrad:
|
||||
case library::ConvKind::kWgrad: return eq_gemm_size(conv_kind).m();
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Workspace used
|
||||
struct Conv2dWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *A;
|
||||
DeviceAllocation *B;
|
||||
DeviceAllocation *C;
|
||||
DeviceAllocation *Computed;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments for convolution operator
|
||||
library::Conv3dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
|
||||
/// Number of copies of the problem workspace which are visited sequentially during
|
||||
/// profiling to avoid camping in the last level cache.
|
||||
int problem_count;
|
||||
|
||||
/// Buffer used for the cutlass conv2d operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
/// Library configuration and arguments for reduction operator
|
||||
library::ReductionConfiguration reduction_configuration;
|
||||
library::ReductionArguments reduction_arguments;
|
||||
|
||||
/// Buffer used for the cutlass reduction operations' host workspace
|
||||
std::vector<uint8_t> reduction_host_workspace;
|
||||
|
||||
/// Host data buffers for host reference operation
|
||||
/// host buffer for tensor
|
||||
std::vector<uint8_t> host_tensor_a;
|
||||
|
||||
/// host buffer for tensor b
|
||||
std::vector<uint8_t> host_tensor_b;
|
||||
|
||||
/// host buffer for tensor c
|
||||
std::vector<uint8_t> host_tensor_c;
|
||||
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
Conv2dWorkspace():
|
||||
A(nullptr), B(nullptr), C(nullptr), Computed(nullptr), Reference(nullptr) { }
|
||||
|
||||
// Returns stride vector for tensor A
|
||||
std::vector<int> stride_a(library::ConvKind const &conv_kind) {
|
||||
return {
|
||||
configuration.layout_a(conv_kind).stride()[0],
|
||||
configuration.layout_a(conv_kind).stride()[1],
|
||||
configuration.layout_a(conv_kind).stride()[2],
|
||||
configuration.layout_a(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor B
|
||||
std::vector<int> stride_b(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_b(conv_kind).stride()[0],
|
||||
configuration.layout_b(conv_kind).stride()[1],
|
||||
configuration.layout_b(conv_kind).stride()[2],
|
||||
configuration.layout_b(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
|
||||
// Returns stride vector for tensor C
|
||||
std::vector<int> stride_c(library::ConvKind const &conv_kind) {
|
||||
|
||||
return {
|
||||
configuration.layout_c(conv_kind).stride()[0],
|
||||
configuration.layout_c(conv_kind).stride()[1],
|
||||
configuration.layout_c(conv_kind).stride()[2],
|
||||
configuration.layout_c(conv_kind).stride()[3]
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// CONV problem obtained from problem space
|
||||
Conv3dProblem problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
Conv2dWorkspace conv_workspace_;
|
||||
|
||||
/// CUTLASS parallel reduction operation to follow this* conv2d operation
|
||||
library::Operation const *reduction_op_;
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
Conv3dOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~Conv3dOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
protected:
|
||||
|
||||
/// Updates the arguments structure for the CUTLASS operator based on
|
||||
/// the problem index.
|
||||
void set_cutlass_operator_arguments_(int problem_idx = 0);
|
||||
|
||||
/// Method to profile an initialized CUTLASS operation
|
||||
virtual Status profile_cutlass_(
|
||||
double &runtime,
|
||||
Options const &options,
|
||||
library::Operation const *operation,
|
||||
void *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace);
|
||||
|
||||
/// Initialize reduction problem dimenstions and library::Operation
|
||||
bool initialize_reduction_configuration_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes the performance result
|
||||
void initialize_result_(
|
||||
PerformanceResult &result,
|
||||
Options const &options,
|
||||
library::ConvDescription const &operation_desc,
|
||||
ProblemSpace const &problem_space);
|
||||
|
||||
/// Verifies CUTLASS against host reference
|
||||
bool verify_with_host_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against device reference
|
||||
bool verify_with_device_reference_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
/// Verifies CUTLASS against cudnn reference
|
||||
bool verify_with_cudnn_(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Helper functions for mapping CUTLASS concepts to cuDNN.
|
||||
*/
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#include "cudnn_helpers.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Converts a cuDNN status to cutlass::Status
|
||||
Status get_cutlass_status(cudnnStatus_t cudnn_status) {
|
||||
|
||||
if (cudnn_status == CUDNN_STATUS_SUCCESS) {
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else if (cudnn_status == CUDNN_STATUS_INVALID_VALUE) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
if (cudnn_status == CUDNN_STATUS_NOT_SUPPORTED) {
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
return Status::kErrorInternal;
|
||||
}
|
||||
|
||||
/// Converts a cuDNN status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cudnnStatus_t cudnn_status) {
|
||||
|
||||
if (cudnn_status == CUDNN_STATUS_INVALID_VALUE) {
|
||||
return Disposition::kInvalidProblem;
|
||||
}
|
||||
else if (cudnn_status == CUDNN_STATUS_NOT_SUPPORTED) {
|
||||
return Disposition::kNotSupported;
|
||||
}
|
||||
return Disposition::kFailed;
|
||||
}
|
||||
|
||||
/// Checks cudnnStatus_t converts to cutlas status and returns if Status::kSuccess o.w. throws exception
|
||||
Status checkCudnnErr(cudnnStatus_t cudnn_status) {
|
||||
Status cutlass_status = get_cutlass_status(cudnn_status);
|
||||
if(cutlass_status != Status::kSuccess) {
|
||||
throw std::runtime_error("checkCudnnErr failed");
|
||||
}
|
||||
return cutlass_status;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS conv mode to a cuDNN cudnnConvolutionMode_t
|
||||
bool get_cudnn_conv_mode(cudnnConvolutionMode_t &cudnn_conv_mode, conv::Mode conv_mode) {
|
||||
switch (conv_mode) {
|
||||
case conv::Mode::kCrossCorrelation:
|
||||
cudnn_conv_mode = CUDNN_CROSS_CORRELATION;
|
||||
return true;
|
||||
case conv::Mode::kConvolution:
|
||||
cudnn_conv_mode = CUDNN_CONVOLUTION;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS tensor layout to a cuDNN cudnnTensorFormat_t
|
||||
bool get_cudnn_layout(cudnnTensorFormat_t &cudnn_layout, library::LayoutTypeID layout) {
|
||||
switch (layout) {
|
||||
// cudnn uses the same enum for TensorNC*HW along nDim (ConvDescription::conv_dim)
|
||||
case library::LayoutTypeID::kTensorNCHW:
|
||||
case library::LayoutTypeID::kTensorNCDHW:
|
||||
cudnn_layout = CUDNN_TENSOR_NCHW;
|
||||
return true;
|
||||
case library::LayoutTypeID::kTensorNHWC:
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
cudnn_layout = CUDNN_TENSOR_NHWC;
|
||||
return true;
|
||||
default: break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps a CUTLASS numeric type to a cuDNN cudnnDataType_t
|
||||
bool get_cudnn_datatype(cudnnDataType_t &cudnn_element_type, library::NumericTypeID element_type) {
|
||||
switch (element_type) {
|
||||
case library::NumericTypeID::kF16:
|
||||
cudnn_element_type = CUDNN_DATA_HALF;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kF32:
|
||||
cudnn_element_type = CUDNN_DATA_FLOAT;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kF64:
|
||||
cudnn_element_type = CUDNN_DATA_DOUBLE;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS2:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS4:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS8:
|
||||
cudnn_element_type = CUDNN_DATA_INT8;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS16:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kS32:
|
||||
cudnn_element_type = CUDNN_DATA_INT32;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kS64:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU2:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU4:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU8:
|
||||
cudnn_element_type = CUDNN_DATA_UINT8;
|
||||
return true;
|
||||
|
||||
case library::NumericTypeID::kU16:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU32:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kU64:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kB1:
|
||||
break;
|
||||
|
||||
case library::NumericTypeID::kInvalid:
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Maps CUTLASS math OpcodeClassID and MathOperationID to cuDNN math_type
|
||||
bool get_cudnn_mathtype(cudnnMathType_t &cudnn_math_type, library::ConvDescription const &conv_desc) {
|
||||
|
||||
switch (conv_desc.tile_description.math_instruction.opcode_class) {
|
||||
|
||||
case library::OpcodeClassID::kTensorOp:
|
||||
{
|
||||
cudnn_math_type = CUDNN_TENSOR_OP_MATH;
|
||||
|
||||
library::MathOperationID math_op = conv_desc.tile_description.math_instruction.math_operation;
|
||||
|
||||
// Allow conversion on input data type for fast math operations
|
||||
if (math_op == library::MathOperationID::kMultiplyAddFastF16 ||
|
||||
math_op == library::MathOperationID::kMultiplyAddFastBF16)
|
||||
{
|
||||
cudnn_math_type = CUDNN_TENSOR_OP_MATH_ALLOW_CONVERSION;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
case library::OpcodeClassID::kSimt:
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Cudnn compute type seems to be hardcoded to float (To handle a possible cudnn issue)
|
||||
float cast_cudnn_compute_type_to_float(library::NumericTypeID type, void const * src) {
|
||||
|
||||
switch (type) {
|
||||
case library::NumericTypeID::kF16:
|
||||
{
|
||||
return float(*(static_cast<half_t const*>(src)));
|
||||
}
|
||||
case library::NumericTypeID::kF32:
|
||||
{
|
||||
return float(*(static_cast<float const*>(src)));
|
||||
}
|
||||
case library::NumericTypeID::kS32:
|
||||
{
|
||||
return float(*(static_cast<int const*>(src)));
|
||||
}
|
||||
default:
|
||||
throw std::runtime_error("Data type handled in cast_compute_type_to_float");
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Returns a status if cuDNN can satisfy a particular Conv2d description
|
||||
Status cudnn_satisfies(
|
||||
library::ConvDescription const &desc,
|
||||
library::Conv2dConfiguration const &configuration) {
|
||||
|
||||
auto const &a_tensor = desc.A;
|
||||
auto const &b_tensor = desc.B;
|
||||
auto const &c_tensor = desc.C;
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if(a_tensor.element != b_tensor.element) {
|
||||
return Status::kErrorInvalidDataType;
|
||||
}
|
||||
|
||||
//////////////////////// Convolution output dimensions p and q ///////////////////////
|
||||
// Cutlass convolutions support arbitrary output dimensions and not constriant by //
|
||||
// input, filter, padding, striding, dilation sizes. //
|
||||
// cuDNN sets the output dimensions (p, q) using following equations: //
|
||||
// //
|
||||
// output = div_up(input + 2 * pad - ((filter - 1) * dilation + 1) + 1, stride) //
|
||||
// where; div_up(a, b) : (a - 1)/b + 1 //
|
||||
// //
|
||||
// Before launching cudnn verification or profiling check that output p and q //
|
||||
// dimensions are cuDNN compliant. //
|
||||
// //
|
||||
// If user sets output p and q which do not follow above constraints, cutlass conv, //
|
||||
// host reference, device reference can run. However, cudnn convolution returns //
|
||||
// "Invalid problem" //
|
||||
// //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// check conv output dimension p for cudnn
|
||||
int cudnn_output_p =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.H +
|
||||
2 * configuration.problem_size.pad_h -
|
||||
((configuration.problem_size.R - 1) *
|
||||
configuration.problem_size.dilation_h + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_h)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_p != configuration.problem_size.P) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension q for cudnn
|
||||
int cudnn_output_q =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.W +
|
||||
2 * configuration.problem_size.pad_w -
|
||||
((configuration.problem_size.S - 1) *
|
||||
configuration.problem_size.dilation_w + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_w)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_q != configuration.problem_size.Q) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// conv operator with input=FP16, accumulator=FP32, output=FP32 datatype
|
||||
if (a_tensor.element == library::NumericTypeID::kF16 &&
|
||||
b_tensor.element == library::NumericTypeID::kF16 &&
|
||||
math_instruction.element_accumulator == library::NumericTypeID::kF32 &&
|
||||
c_tensor.element == library::NumericTypeID::kF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
b_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
c_tensor.element == library::NumericTypeID::kBF16
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// TF32 input not supported in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
b_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
c_tensor.element == library::NumericTypeID::kTF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS8 ||
|
||||
b_tensor.element == library::NumericTypeID::kS8 ||
|
||||
c_tensor.element == library::NumericTypeID::kS8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kU8 ||
|
||||
b_tensor.element == library::NumericTypeID::kU8 ||
|
||||
c_tensor.element == library::NumericTypeID::kU8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS4 ||
|
||||
b_tensor.element == library::NumericTypeID::kS4 ||
|
||||
c_tensor.element == library::NumericTypeID::kS4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kU4 ||
|
||||
b_tensor.element == library::NumericTypeID::kU4 ||
|
||||
c_tensor.element == library::NumericTypeID::kU4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns a status if cuDNN can satisfy a particular Conv3d description
|
||||
Status cudnn_satisfies(
|
||||
library::ConvDescription const &desc,
|
||||
library::Conv3dConfiguration const &configuration) {
|
||||
|
||||
auto const &a_tensor = desc.A;
|
||||
auto const &b_tensor = desc.B;
|
||||
auto const &c_tensor = desc.C;
|
||||
auto const &math_instruction = desc.tile_description.math_instruction;
|
||||
|
||||
if(a_tensor.element != b_tensor.element) {
|
||||
return Status::kErrorInvalidDataType;
|
||||
}
|
||||
|
||||
//////////////////////// Convolution output dimensions p and q ///////////////////////
|
||||
// Cutlass convolutions support arbitrary output dimensions and not constriant by //
|
||||
// input, filter, padding, striding, dilation sizes. //
|
||||
// cuDNN sets the output dimensions (p, q) using following equations: //
|
||||
// //
|
||||
// output = div_up(input + 2 * pad - ((filter - 1) * dilation + 1) + 1, stride) //
|
||||
// where; div_up(a, b) : (a - 1)/b + 1 //
|
||||
// //
|
||||
// Before launching cudnn verification or profiling check that output p and q //
|
||||
// dimensions are cuDNN compliant. //
|
||||
// //
|
||||
// If user sets output p and q which do not follow above constraints, cutlass conv, //
|
||||
// host reference, device reference can run. However, cudnn convolution returns //
|
||||
// "Invalid problem" //
|
||||
// //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// check conv output dimension z for cudnn
|
||||
int cudnn_output_z =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.D +
|
||||
2 * configuration.problem_size.pad_d -
|
||||
((configuration.problem_size.T - 1) *
|
||||
configuration.problem_size.dilation_d + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_d)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_z != configuration.problem_size.Z) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension p for cudnn
|
||||
int cudnn_output_p =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.H +
|
||||
2 * configuration.problem_size.pad_h -
|
||||
((configuration.problem_size.R - 1) *
|
||||
configuration.problem_size.dilation_h + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_h)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_p != configuration.problem_size.P) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
// check conv output dimension q for cudnn
|
||||
int cudnn_output_q =
|
||||
(
|
||||
(
|
||||
configuration.problem_size.W +
|
||||
2 * configuration.problem_size.pad_w -
|
||||
((configuration.problem_size.S - 1) *
|
||||
configuration.problem_size.dilation_w + 1)
|
||||
) /
|
||||
(configuration.problem_size.stride_w)
|
||||
+ 1
|
||||
);
|
||||
|
||||
if (cudnn_output_q != configuration.problem_size.Q) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// conv operator with input, accumulator, output datatype of (hss) are not supported
|
||||
// in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kF16 &&
|
||||
b_tensor.element == library::NumericTypeID::kF16 &&
|
||||
math_instruction.element_accumulator == library::NumericTypeID::kF32 &&
|
||||
c_tensor.element == library::NumericTypeID::kF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
b_tensor.element == library::NumericTypeID::kBF16 ||
|
||||
c_tensor.element == library::NumericTypeID::kBF16
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
b_tensor.element == library::NumericTypeID::kTF32 ||
|
||||
c_tensor.element == library::NumericTypeID::kTF32
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
if (a_tensor.element == library::NumericTypeID::kS8 ||
|
||||
b_tensor.element == library::NumericTypeID::kS8 ||
|
||||
c_tensor.element == library::NumericTypeID::kS8
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// S4 not supported in cuDNN
|
||||
if (a_tensor.element == library::NumericTypeID::kS4 ||
|
||||
b_tensor.element == library::NumericTypeID::kS4 ||
|
||||
c_tensor.element == library::NumericTypeID::kS4
|
||||
) {
|
||||
|
||||
return Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,584 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Helper functions for mapping CUTLASS concepts to cuDNN.
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include <cuda_runtime.h>
|
||||
#include <cudnn.h>
|
||||
#include <iostream>
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/util/device_memory.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "enumerated_types.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Converts a cuDNN status to cutlass::Status
|
||||
Status get_cutlass_status(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Converts a cuDNN status to cutlass::profiler::Disposition
|
||||
Disposition get_cutlass_disposition(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Checks cudnnStatus_t converts to cutlas status and returns if Status::kSuccess o.w. throws exception
|
||||
Status checkCudnnErr(cudnnStatus_t cudnn_status);
|
||||
|
||||
/// Maps a CUTLASS conv mode to a cuDNN conv mode enumeration
|
||||
bool get_cudnn_conv_mode(cudnnConvolutionMode_t &cudnn_conv_mode, conv::Mode conv_mode);
|
||||
|
||||
/// Maps a CUTLASS layout type to a cuDNN data type enumeration
|
||||
bool get_cudnn_layout(cudnnTensorFormat_t &cudnn_layout, library::LayoutTypeID layout);
|
||||
|
||||
/// Maps a CUTLASS numeric type to a cuDNN data type enumeration
|
||||
bool get_cudnn_datatype(cudnnDataType_t &cudnn_element_type, library::NumericTypeID element_type);
|
||||
|
||||
/// Maps CUTLASS math OpcodeClassID and MathOperationID to cuDNN math_type
|
||||
bool get_cudnn_mathtype(cudnnMathType_t &cudnn_math_type, library::ConvDescription const &conv_desc);
|
||||
|
||||
/// Returns a status if cudnn can satisfy a particular Conv2d description
|
||||
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv2dConfiguration const &configuration);
|
||||
|
||||
/// Returns a status if cudnn can satisfy a particular Conv3d description
|
||||
Status cudnn_satisfies(library::ConvDescription const &desc, library::Conv3dConfiguration const &configuration);
|
||||
|
||||
/// Cudnn compute type seems to be hardcoded to float (To handle a possible cudnn issue)
|
||||
float cast_cudnn_compute_type_to_float(library::NumericTypeID type, void const * src);
|
||||
|
||||
|
||||
/// This is a helper class to create cudnnHandle_t automatically on CudnnCreate object creation and
|
||||
/// to destroy cudnnHandle_t on CudnnCreate object destruction.
|
||||
/// Additionaly, it provides implicit cast from CudnnCreate's object to cudnnHandle_t's object
|
||||
class CudnnCreate {
|
||||
private:
|
||||
cudnnHandle_t handle;
|
||||
cudnnStatus_t status;
|
||||
|
||||
public:
|
||||
CudnnCreate() {
|
||||
status = cudnnCreate(&handle);
|
||||
}
|
||||
|
||||
~CudnnCreate() {
|
||||
cudnnDestroy(handle);
|
||||
}
|
||||
|
||||
/// Implicit cast CudnnCreate object to cudnnHandle_t
|
||||
operator cudnnHandle_t() const { return handle; }
|
||||
|
||||
/// returns cudnnStatus_t for handle creation
|
||||
cudnnStatus_t get_cudnn_create_status() { return status; }
|
||||
};
|
||||
|
||||
|
||||
namespace detail {
|
||||
|
||||
/// Dispatcher to cudnn convolution operators
|
||||
struct cudnnConvDispatcher {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
//library::Conv2dConfiguration configuration;
|
||||
library::ConvArguments arguments;
|
||||
library::ConvKind conv_kind;
|
||||
|
||||
// cudnn-specific data structures to fill cudnn API call arguments
|
||||
// cudnn activation, filter, and output descriptors
|
||||
cudnnTensorDescriptor_t activation_desc;
|
||||
cudnnFilterDescriptor_t filter_desc;
|
||||
cudnnTensorDescriptor_t output_desc;
|
||||
cudnnConvolutionDescriptor_t conv_desc;
|
||||
|
||||
// cudnn datatypes
|
||||
cudnnDataType_t data_type_activation;
|
||||
cudnnDataType_t data_type_filter;
|
||||
cudnnDataType_t data_type_output;
|
||||
|
||||
// cudnn layouts
|
||||
cudnnTensorFormat_t layout_activation;
|
||||
cudnnTensorFormat_t layout_filter;
|
||||
cudnnTensorFormat_t layout_output;
|
||||
|
||||
// cudnn convolution mode
|
||||
cudnnConvolutionMode_t conv_mode;
|
||||
|
||||
// cudnn math type (tensorop, tensorop with conversion, simt)
|
||||
cudnnMathType_t math_type;
|
||||
|
||||
// cudnn compute data type
|
||||
cudnnDataType_t compute_type;
|
||||
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
float alpha;
|
||||
float beta;
|
||||
|
||||
// cudnn workspace
|
||||
size_t workspace_size_in_bytes = 0;
|
||||
cutlass::device_memory::allocation<char> workspace;
|
||||
|
||||
// select cudnn's implicit gemm precomputed algorithm with tensor operations
|
||||
static cudnnConvolutionFwdAlgo_t const fprop_algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM;
|
||||
static cudnnConvolutionBwdDataAlgo_t const dgrad_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1;
|
||||
static cudnnConvolutionBwdFilterAlgo_t const wgrad_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1;
|
||||
|
||||
Status status;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// TODO: unify ctor cudnnConvDispatcher for conv2d and conv3d by unifying Conv2dConfigration
|
||||
|
||||
// ctor for conv2d
|
||||
cudnnConvDispatcher(
|
||||
library::ConvDescription const &op_desc,
|
||||
library::Conv2dConfiguration configuration,
|
||||
library::ConvArguments arguments_,
|
||||
cudnnHandle_t handle
|
||||
):
|
||||
//configuration(configuration_),
|
||||
arguments(arguments_),
|
||||
conv_kind(op_desc.conv_kind),
|
||||
status(Status::kSuccess) {
|
||||
|
||||
bool good = true;
|
||||
|
||||
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
|
||||
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
|
||||
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
|
||||
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
|
||||
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
|
||||
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
|
||||
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
|
||||
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
|
||||
// Get cudnn mathtype (cudnnMathType_t)
|
||||
good = (good && get_cudnn_mathtype(math_type, op_desc));
|
||||
good = (good && get_cudnn_datatype(
|
||||
compute_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
// Check cutlass Conv2d description has equivalent operator in cudnn
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
return;
|
||||
}
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
|
||||
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
|
||||
|
||||
// Create convolution descriptor object
|
||||
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
|
||||
|
||||
// Configure convolution operator
|
||||
std::vector<int> padding {configuration.problem_size.pad_h, configuration.problem_size.pad_w};
|
||||
std::vector<int> stride {configuration.problem_size.stride_h, configuration.problem_size.stride_w};
|
||||
std::vector<int> dilation {configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionNdDescriptor(
|
||||
conv_desc,
|
||||
op_desc.conv_dim,
|
||||
padding.data(),
|
||||
stride.data(),
|
||||
dilation.data(),
|
||||
conv_mode,
|
||||
compute_type
|
||||
));
|
||||
|
||||
// Set groups
|
||||
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
|
||||
|
||||
// Create activation, filter, and output descriptor objects
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
|
||||
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
|
||||
|
||||
// Set activation, filter, and output descriptor
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensor4dDescriptor(
|
||||
activation_desc,
|
||||
layout_activation,
|
||||
data_type_activation,
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.H,
|
||||
configuration.problem_size.W
|
||||
));
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetFilter4dDescriptor(
|
||||
filter_desc,
|
||||
data_type_filter,
|
||||
layout_filter,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.R,
|
||||
configuration.problem_size.S
|
||||
));
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensor4dDescriptor(
|
||||
output_desc,
|
||||
layout_output,
|
||||
data_type_output,
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.P,
|
||||
configuration.problem_size.Q
|
||||
));
|
||||
|
||||
// Set math instruction to tensor op
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionMathType(conv_desc, math_type));
|
||||
|
||||
// Initialize workspace
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionForwardWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
filter_desc,
|
||||
conv_desc,
|
||||
output_desc,
|
||||
fprop_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kDgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardDataWorkspaceSize(
|
||||
handle,
|
||||
filter_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
activation_desc,
|
||||
dgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kWgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardFilterWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
filter_desc,
|
||||
wgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
|
||||
}
|
||||
|
||||
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
|
||||
}
|
||||
|
||||
|
||||
// ctor for conv3d
|
||||
cudnnConvDispatcher(
|
||||
library::ConvDescription const &op_desc,
|
||||
library::Conv3dConfiguration configuration,
|
||||
library::ConvArguments arguments_,
|
||||
cudnnHandle_t handle
|
||||
):
|
||||
//configuration(configuration_),
|
||||
arguments(arguments_),
|
||||
conv_kind(op_desc.conv_kind),
|
||||
status(Status::kSuccess) {
|
||||
|
||||
bool good = true;
|
||||
|
||||
// Get cudnn datatype, layout, and convolution mode from library::ConvDescription
|
||||
good = (good && get_cudnn_datatype(data_type_activation, op_desc.A.element));
|
||||
good = (good && get_cudnn_datatype(data_type_filter, op_desc.B.element));
|
||||
good = (good && get_cudnn_datatype(data_type_output, op_desc.C.element));
|
||||
|
||||
good = (good && get_cudnn_layout(layout_activation, op_desc.A.layout));
|
||||
good = (good && get_cudnn_layout(layout_filter, op_desc.B.layout));
|
||||
good = (good && get_cudnn_layout(layout_output, op_desc.C.layout));
|
||||
|
||||
good = (good && get_cudnn_conv_mode(conv_mode, configuration.problem_size.mode));
|
||||
|
||||
// cudnn compute type seems to be hardcoded to float (to handle a possible a cudnn issue)
|
||||
alpha = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.alpha);
|
||||
beta = cast_cudnn_compute_type_to_float(op_desc.element_epilogue, arguments.beta);
|
||||
|
||||
good = (good && get_cudnn_datatype(
|
||||
compute_type,
|
||||
op_desc.tile_description.math_instruction.element_accumulator));
|
||||
|
||||
// Check cutlass Conv2d description has equivalent operator in cudnn
|
||||
if (!good) {
|
||||
status = Status::kErrorNotSupported;
|
||||
}
|
||||
|
||||
// Create convolution descriptor object
|
||||
status = get_cutlass_status(cudnnCreateConvolutionDescriptor(&conv_desc));
|
||||
|
||||
// Configure convolution operator
|
||||
std::vector<int> padding {configuration.problem_size.pad_d, configuration.problem_size.pad_h, configuration.problem_size.pad_w};
|
||||
std::vector<int> stride {configuration.problem_size.stride_d, configuration.problem_size.stride_h, configuration.problem_size.stride_w};
|
||||
std::vector<int> dilation {configuration.problem_size.dilation_d, configuration.problem_size.dilation_h, configuration.problem_size.dilation_w};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionNdDescriptor(
|
||||
conv_desc,
|
||||
op_desc.conv_dim,
|
||||
padding.data(),
|
||||
stride.data(),
|
||||
dilation.data(),
|
||||
conv_mode,
|
||||
compute_type
|
||||
));
|
||||
|
||||
// Set groups
|
||||
status = get_cutlass_status(cudnnSetConvolutionGroupCount(conv_desc, configuration.problem_size.groups));
|
||||
|
||||
// Create activation, filter, and output descriptor objects
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&activation_desc));
|
||||
status = get_cutlass_status(cudnnCreateFilterDescriptor(&filter_desc));
|
||||
status = get_cutlass_status(cudnnCreateTensorDescriptor(&output_desc));
|
||||
|
||||
// Set activation descriptor
|
||||
std::vector<int> activation_extent {
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.D,
|
||||
configuration.problem_size.H,
|
||||
configuration.problem_size.W
|
||||
};
|
||||
|
||||
std::vector<int> activation_stride {
|
||||
configuration.layout_activations.stride()[3],
|
||||
1,
|
||||
configuration.layout_activations.stride()[2],
|
||||
configuration.layout_activations.stride()[1],
|
||||
configuration.layout_activations.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensorNdDescriptor(
|
||||
activation_desc,
|
||||
data_type_activation,
|
||||
op_desc.conv_dim + 2,
|
||||
activation_extent.data(),
|
||||
activation_stride.data()
|
||||
));
|
||||
|
||||
// Set filter descriptor
|
||||
std::vector<int> filter_extent {
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.C,
|
||||
configuration.problem_size.T,
|
||||
configuration.problem_size.R,
|
||||
configuration.problem_size.S
|
||||
};
|
||||
|
||||
std::vector<int> filter_stride {
|
||||
configuration.layout_filters.stride()[3],
|
||||
1,
|
||||
configuration.layout_filters.stride()[2],
|
||||
configuration.layout_filters.stride()[1],
|
||||
configuration.layout_filters.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetFilterNdDescriptor(
|
||||
filter_desc,
|
||||
data_type_filter,
|
||||
layout_filter,
|
||||
op_desc.conv_dim + 2,
|
||||
filter_extent.data()
|
||||
));
|
||||
|
||||
|
||||
// Set output descriptor
|
||||
std::vector<int> output_extent {
|
||||
configuration.problem_size.N,
|
||||
configuration.problem_size.K,
|
||||
configuration.problem_size.Z,
|
||||
configuration.problem_size.P,
|
||||
configuration.problem_size.Q
|
||||
};
|
||||
|
||||
std::vector<int> output_stride {
|
||||
configuration.layout_output.stride()[3],
|
||||
1,
|
||||
configuration.layout_output.stride()[2],
|
||||
configuration.layout_output.stride()[1],
|
||||
configuration.layout_output.stride()[0]
|
||||
};
|
||||
|
||||
status = get_cutlass_status(
|
||||
cudnnSetTensorNdDescriptor(
|
||||
output_desc,
|
||||
data_type_output,
|
||||
op_desc.conv_dim + 2,
|
||||
output_extent.data(),
|
||||
output_stride.data()
|
||||
));
|
||||
|
||||
// Set math instruction to tensor op
|
||||
status = get_cutlass_status(
|
||||
cudnnSetConvolutionMathType(conv_desc, math_type));
|
||||
|
||||
// Initialize workspace
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionForwardWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
filter_desc,
|
||||
conv_desc,
|
||||
output_desc,
|
||||
fprop_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kDgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardDataWorkspaceSize(
|
||||
handle,
|
||||
filter_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
activation_desc,
|
||||
dgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
case library::ConvKind::kWgrad:
|
||||
status = get_cutlass_status(
|
||||
cudnnGetConvolutionBackwardFilterWorkspaceSize(
|
||||
handle,
|
||||
activation_desc,
|
||||
output_desc,
|
||||
conv_desc,
|
||||
filter_desc,
|
||||
wgrad_algo,
|
||||
&workspace_size_in_bytes
|
||||
)); break;
|
||||
|
||||
}
|
||||
|
||||
workspace = cutlass::device_memory::allocation<char>(workspace_size_in_bytes);
|
||||
}
|
||||
|
||||
/// Executes Conv2d operater from cudnn library
|
||||
cudnnStatus_t operator()(cudnnHandle_t handle) {
|
||||
|
||||
switch (conv_kind) {
|
||||
case library::ConvKind::kFprop:
|
||||
return cudnnConvolutionForward(
|
||||
handle,
|
||||
&alpha,
|
||||
activation_desc,
|
||||
activation(),
|
||||
filter_desc,
|
||||
filter(),
|
||||
conv_desc,
|
||||
fprop_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
output_desc,
|
||||
arguments.D
|
||||
);
|
||||
case library::ConvKind::kDgrad:
|
||||
return cudnnConvolutionBackwardData(
|
||||
handle,
|
||||
&alpha,
|
||||
filter_desc,
|
||||
filter(),
|
||||
output_desc,
|
||||
output(),
|
||||
conv_desc,
|
||||
dgrad_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
activation_desc,
|
||||
arguments.D
|
||||
);
|
||||
case library::ConvKind::kWgrad:
|
||||
return cudnnConvolutionBackwardFilter(
|
||||
handle,
|
||||
&alpha,
|
||||
activation_desc,
|
||||
activation(),
|
||||
output_desc,
|
||||
output(),
|
||||
conv_desc,
|
||||
wgrad_algo,
|
||||
workspace.get(),
|
||||
workspace_size_in_bytes,
|
||||
&beta,
|
||||
filter_desc,
|
||||
arguments.D
|
||||
);
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Actviation Tensor
|
||||
void const * activation() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.A;
|
||||
case library::ConvKind::kDgrad : return arguments.C;
|
||||
case library::ConvKind::kWgrad : return arguments.B;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Filter Tensor
|
||||
void const *filter() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.B;
|
||||
case library::ConvKind::kDgrad : return arguments.B;
|
||||
case library::ConvKind::kWgrad : return arguments.C;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
|
||||
// Returns Output Tensor
|
||||
void const *output() const {
|
||||
switch(conv_kind) {
|
||||
case library::ConvKind::kFprop : return arguments.C;
|
||||
case library::ConvKind::kDgrad : return arguments.A;
|
||||
case library::ConvKind::kWgrad : return arguments.A;
|
||||
default : throw std::runtime_error("Invalid Conv Operator (fprop, dgrad, wgrad)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
@@ -32,6 +32,8 @@
|
||||
// Profiler includes
|
||||
#include "cutlass_profiler.h"
|
||||
#include "gemm_operation_profiler.h"
|
||||
#include "conv2d_operation_profiler.h"
|
||||
#include "conv3d_operation_profiler.h"
|
||||
#include "sparse_gemm_operation_profiler.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -50,6 +52,10 @@ CutlassProfiler::CutlassProfiler(
|
||||
|
||||
operation_profilers_.emplace_back(new SparseGemmOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new Conv2dOperationProfiler(options));
|
||||
|
||||
operation_profilers_.emplace_back(new Conv3dOperationProfiler(options));
|
||||
|
||||
}
|
||||
|
||||
CutlassProfiler::~CutlassProfiler() {
|
||||
@@ -159,6 +165,8 @@ void CutlassProfiler::print_usage_(std::ostream &out) {
|
||||
|
||||
out << "\n\nFor details about a particular function, specify the function name with --help.\n\nExample:\n\n"
|
||||
<< " $ cutlass_profiler --operation=Gemm --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv3d --help\n\n"
|
||||
<< " $ cutlass_profiler --operation=Conv2d --help\n\n"
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +133,18 @@ std::vector<int> DeviceAllocation::get_packed_layout(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNDHWC>(extent);
|
||||
break;
|
||||
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNCxHWx<32>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorNCxHWx<64>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorCxRSKx<32>>(extent);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
stride = get_packed_layout_stride<cutlass::layout::TensorCxRSKx<64>>(extent);
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -247,6 +258,18 @@ size_t DeviceAllocation::construct_layout(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
return construct_layout_<cutlass::layout::TensorNDHWC>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
return construct_layout_<cutlass::layout::TensorNCxHWx<32>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
return construct_layout_<cutlass::layout::TensorNCxHWx<64>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
return construct_layout_<cutlass::layout::TensorCxRSKx<32>>(bytes, layout_id, extent, stride);
|
||||
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
return construct_layout_<cutlass::layout::TensorCxRSKx<64>>(bytes, layout_id, extent, stride);
|
||||
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -1362,6 +1385,18 @@ static void write_tensor_csv_static_type(
|
||||
case library::LayoutTypeID::kTensorNDHWC:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNDHWC>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC32HW32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNCxHWx<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorNC64HW64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorNCxHWx<64>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC32RSK32:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorCxRSKx<32>>(out, allocation);
|
||||
break;
|
||||
case library::LayoutTypeID::kTensorC64RSK64:
|
||||
write_tensor_csv_static_tensor_view<T, layout::TensorCxRSKx<64>>(out, allocation);
|
||||
break;
|
||||
default:
|
||||
throw std::runtime_error("Unhandled layout");
|
||||
}
|
||||
|
||||
@@ -243,7 +243,7 @@ int OperationProfiler::profile_all(
|
||||
ProblemSpace::Iterator problem_it = problem_space.begin();
|
||||
ProblemSpace::Iterator problem_end = problem_space.end();
|
||||
|
||||
bool continue_profiling = true;
|
||||
bool continue_profiling = true, internal_error = false;
|
||||
|
||||
// For each problem in problem space
|
||||
for (; continue_profiling && problem_it != problem_end; ++problem_it) {
|
||||
@@ -302,7 +302,8 @@ int OperationProfiler::profile_all(
|
||||
|
||||
if (status == Status::kErrorInternal) {
|
||||
// Stop profiling if there was an internal error
|
||||
return false;
|
||||
internal_error = true;
|
||||
break;
|
||||
}
|
||||
else if (status != Status::kSuccess) {
|
||||
// If the workspace could not be initialized for any other reason, continue to
|
||||
@@ -322,7 +323,8 @@ int OperationProfiler::profile_all(
|
||||
|
||||
if (status == Status::kErrorInternal) {
|
||||
// Stop profiling if there was an internal error
|
||||
return false;
|
||||
internal_error = true;
|
||||
break;
|
||||
}
|
||||
else if (status != Status::kSuccess) {
|
||||
// If the workspace could not be initialized for any other reason, continue to
|
||||
@@ -336,8 +338,9 @@ int OperationProfiler::profile_all(
|
||||
//
|
||||
|
||||
// B. Verify CUTLASS
|
||||
if (continue_profiling) {
|
||||
|
||||
|
||||
if (continue_profiling && options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
continue_profiling = this->verify_cutlass(
|
||||
options,
|
||||
report,
|
||||
@@ -368,6 +371,7 @@ int OperationProfiler::profile_all(
|
||||
//
|
||||
// D. Profile
|
||||
//
|
||||
|
||||
if (continue_profiling && options.profiling.enabled) {
|
||||
|
||||
continue_profiling = this->profile(
|
||||
@@ -392,10 +396,7 @@ int OperationProfiler::profile_all(
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Emit report
|
||||
report.close();
|
||||
|
||||
return 0;
|
||||
return internal_error ? 1 : 0;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -401,6 +401,7 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
|
||||
else {
|
||||
providers.push_back(library::Provider::kCUTLASS);
|
||||
providers.push_back(library::Provider::kCUBLAS);
|
||||
providers.push_back(library::Provider::kCUDNN);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,8 +429,8 @@ void Options::Profiling::print_usage(std::ostream &out) const {
|
||||
|
||||
<< " --providers=<providers> "
|
||||
<< " List of providers to be profiled for performance. (default: '*')" << end_of_line
|
||||
<< " Gemm providers {cutlass*"
|
||||
<< "}" << end_of_line
|
||||
<< " Gemm providers {cutlass*, cublas*}" << end_of_line
|
||||
<< " Conv2d providers {cutlass*, cudnn*}"
|
||||
<< "\n\n";
|
||||
|
||||
}
|
||||
@@ -502,6 +503,7 @@ Options::Verification::Verification(cutlass::CommandLine const &cmdline) {
|
||||
else {
|
||||
providers.push_back(library::Provider::kCUBLAS);
|
||||
providers.push_back(library::Provider::kReferenceDevice);
|
||||
providers.push_back(library::Provider::kCUDNN);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -529,6 +531,7 @@ void Options::Verification::print_usage(std::ostream &out) const {
|
||||
<< " --verification-providers=<providers> "
|
||||
<< " List of providers used to verify result. (default: '*')" << end_of_line
|
||||
<< " Gemm verification-providers {cublas*}" << end_of_line
|
||||
<< " Conv2d verification-providers {cudnn*, device*, host}"
|
||||
<< "\n\n";
|
||||
}
|
||||
|
||||
@@ -570,6 +573,7 @@ Options::Report::Report(cutlass::CommandLine const &cmdline) {
|
||||
|
||||
cmdline.get_cmd_line_argument("append", append, false);
|
||||
cmdline.get_cmd_line_argument("output", output_path);
|
||||
cmdline.get_cmd_line_argument("junit-output", junit_output_path);
|
||||
|
||||
if (cmdline.check_cmd_line_flag("tags")) {
|
||||
cmdline.get_cmd_line_argument_pairs("tags", pivot_tags);
|
||||
@@ -591,6 +595,9 @@ void Options::Report::print_usage(std::ostream &out) const {
|
||||
<< " --output=<path> "
|
||||
<< " Path to output file for machine readable results. Operation kind and '.csv' is appended.\n\n"
|
||||
|
||||
<< " --junit-output=<path> "
|
||||
<< " Path to junit output file for result reporting. Operation kind and '.junit.xml' is appended.\n\n"
|
||||
|
||||
<< " --report-not-run=<bool> "
|
||||
<< " If true, reports the status of all kernels including those that" << end_of_line
|
||||
<< " do not satisfy the given arguments.\n\n"
|
||||
@@ -608,6 +615,7 @@ void Options::Report::print_options(std::ostream &out, int indent) const {
|
||||
out
|
||||
<< indent_str(indent) << "append: " << append << "\n"
|
||||
<< indent_str(indent) << "output: " << output_path << "\n"
|
||||
<< indent_str(indent) << "junit-output: " << junit_output_path << "\n"
|
||||
<< indent_str(indent) << "report_not_run: " << report_not_run << "\n"
|
||||
<< indent_str(indent) << "tags:\n";
|
||||
|
||||
|
||||
@@ -218,6 +218,9 @@ public:
|
||||
/// Path to a file containing results
|
||||
std::string output_path;
|
||||
|
||||
/// Path to a file containing junit xml results
|
||||
std::string junit_output_path;
|
||||
|
||||
/// Sequence of tags to attach to each result
|
||||
std::vector<std::pair<std::string, std::string>> pivot_tags;
|
||||
|
||||
|
||||
@@ -69,11 +69,15 @@ PerformanceReport::PerformanceReport(
|
||||
options_(options), argument_names_(argument_names), problem_index_(0), good_(true), op_kind_(op_kind) {
|
||||
|
||||
// Strip '.csv' if present
|
||||
std::string base_path = options_.report.output_path.substr(
|
||||
0, options_.report.output_path.rfind(".csv"));
|
||||
|
||||
std::string base_path = options_.report.output_path;
|
||||
base_path = base_path.substr(0, base_path.rfind(".csv"));
|
||||
op_file_name_ = base_path + "." + to_string(op_kind_) + ".csv";
|
||||
|
||||
base_path = options_.report.junit_output_path;
|
||||
base_path = base_path.substr(0, base_path.rfind(".xml"));
|
||||
base_path = base_path.substr(0, base_path.rfind(".junit"));
|
||||
op_junit_file_name_ = base_path + "." + to_string(op_kind_) + ".junit.xml";
|
||||
|
||||
//
|
||||
// Open output file for operation of PerformanceReport::op_kind
|
||||
//
|
||||
@@ -108,6 +112,21 @@ PerformanceReport::PerformanceReport(
|
||||
print_csv_header_(output_file_) << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
if (!options_.report.junit_output_path.empty()) {
|
||||
|
||||
junit_output_file_.open(op_junit_file_name_);
|
||||
|
||||
if (!junit_output_file_.good()) {
|
||||
|
||||
std::cerr << "Could not open junit output file at path '"
|
||||
<< options_.report.junit_output_path << "'" << std::endl;
|
||||
|
||||
good_ = false;
|
||||
}
|
||||
|
||||
print_junit_header_(junit_output_file_);
|
||||
}
|
||||
}
|
||||
|
||||
void PerformanceReport::next_problem() {
|
||||
@@ -123,6 +142,10 @@ void PerformanceReport::append_result(PerformanceResult result) {
|
||||
print_result_pretty_(std::cout, result) << std::flush;
|
||||
}
|
||||
|
||||
if (junit_output_file_.is_open()) {
|
||||
print_junit_result_(junit_output_file_, result);
|
||||
}
|
||||
|
||||
if (output_file_.is_open()) {
|
||||
print_result_csv_(output_file_, result) << std::endl;
|
||||
}
|
||||
@@ -143,7 +166,7 @@ void PerformanceReport::append_results(PerformanceResultVector const &results) {
|
||||
}
|
||||
}
|
||||
|
||||
void PerformanceReport::close() {
|
||||
PerformanceReport::~PerformanceReport() {
|
||||
|
||||
//
|
||||
// Output results to stdout if they were not written to a file already.
|
||||
@@ -161,7 +184,17 @@ void PerformanceReport::close() {
|
||||
}
|
||||
}
|
||||
else if (output_file_.is_open() && options_.report.verbose) {
|
||||
std::cout << "\n\nWrote results to '" << op_file_name_ << "'" << std::endl;
|
||||
std::cout << "\nWrote results to '" << op_file_name_ << "'" << std::endl;
|
||||
}
|
||||
|
||||
if (output_file_.is_open()) {
|
||||
output_file_.close();
|
||||
}
|
||||
|
||||
if (junit_output_file_.is_open()) {
|
||||
print_junit_footer_(junit_output_file_);
|
||||
junit_output_file_.close();
|
||||
std::cout << "\nWrote jUnit results to '" << op_junit_file_name_ << "'" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +212,8 @@ static const char *disposition_status_color(Disposition disposition) {
|
||||
/// Prints the result in human readable form
|
||||
std::ostream & PerformanceReport::print_result_pretty_(
|
||||
std::ostream &out,
|
||||
PerformanceResult const &result) {
|
||||
PerformanceResult const &result,
|
||||
bool use_shell_coloring) {
|
||||
|
||||
out << "=============================\n"
|
||||
<< " Problem ID: " << result.problem_index << "\n";
|
||||
@@ -196,14 +230,20 @@ std::ostream & PerformanceReport::print_result_pretty_(
|
||||
out << "\n";
|
||||
}
|
||||
|
||||
std::string shell_color_bright = use_shell_coloring ? SHELL_COLOR_BRIGHT() : "";
|
||||
std::string shell_color_end = use_shell_coloring ? SHELL_COLOR_END() : "";
|
||||
auto _disposition_status_color = [&](Disposition d) -> const char * {
|
||||
return use_shell_coloring ? disposition_status_color(d) : "";
|
||||
};
|
||||
|
||||
out
|
||||
<< "\n"
|
||||
<< " Provider: " << SHELL_COLOR_BRIGHT() << library::to_string(result.provider, true) << SHELL_COLOR_END() << "\n"
|
||||
<< " OperationKind: " << SHELL_COLOR_BRIGHT() << library::to_string(result.op_kind) << SHELL_COLOR_END() << "\n"
|
||||
<< " Provider: " << shell_color_bright << library::to_string(result.provider, true) << shell_color_end << "\n"
|
||||
<< " OperationKind: " << shell_color_bright << library::to_string(result.op_kind) << shell_color_end << "\n"
|
||||
<< " Operation: " << result.operation_name << "\n\n"
|
||||
<< " Status: " << SHELL_COLOR_BRIGHT() << library::to_string(result.status, true) << SHELL_COLOR_END() << "\n"
|
||||
<< " Verification: " << SHELL_COLOR_BRIGHT() << (options_.verification.enabled ? "ON":"OFF") << SHELL_COLOR_END() << "\n"
|
||||
<< " Disposition: " << disposition_status_color(result.disposition) << to_string(result.disposition, true) << SHELL_COLOR_END() << "\n\n";
|
||||
<< " Status: " << shell_color_bright << library::to_string(result.status, true) << shell_color_end << "\n"
|
||||
<< " Verification: " << shell_color_bright << (options_.verification.enabled ? "ON":"OFF") << shell_color_end << "\n"
|
||||
<< " Disposition: " << _disposition_status_color(result.disposition) << to_string(result.disposition, true) << shell_color_end << "\n\n";
|
||||
|
||||
// Display individual verification results for each verification-provider
|
||||
if (options_.verification.enabled) {
|
||||
@@ -263,10 +303,6 @@ std::ostream & PerformanceReport::print_csv_header_(
|
||||
<< ",OperationKind,Operation,Disposition,Status";
|
||||
|
||||
for (auto const &arg_name : argument_names_) {
|
||||
// Operand E is internal to the sparse kernel
|
||||
if (arg_name.compare("E") == 0)
|
||||
continue;
|
||||
|
||||
out << "," << arg_name;
|
||||
}
|
||||
|
||||
@@ -327,6 +363,112 @@ std::ostream & PerformanceReport::print_result_csv_(
|
||||
return out;
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_header_(std::ostream &out) {
|
||||
|
||||
out << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" << std::endl;
|
||||
out << "<testsuite name=\"cutlass_profiler\">" << std::endl;
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
std::string escape_xml_special_chars(const std::string& src) {
|
||||
std::stringstream dst;
|
||||
for (char ch : src) {
|
||||
switch (ch) {
|
||||
case '&': dst << "&"; break;
|
||||
case '\'': dst << "'"; break;
|
||||
case '"': dst << """; break;
|
||||
case '<': dst << "<"; break;
|
||||
case '>': dst << ">"; break;
|
||||
default: dst << ch; break;
|
||||
}
|
||||
}
|
||||
return dst.str();
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
std::ostream & print_junit_result_property_(std::ostream & os, const std::string & name, const T & property) {
|
||||
return os << " <property name=\"" << name << "\" value=\"" << property << "\" />" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_result_(std::ostream &out, PerformanceResult const &result) {
|
||||
|
||||
out << " " << "<testcase name=\"";
|
||||
|
||||
std::string delim = "";
|
||||
|
||||
// Pivot tags
|
||||
for (auto const & tag : options_.report.pivot_tags) {
|
||||
out << delim << tag.second; delim = "_";
|
||||
}
|
||||
|
||||
out << delim << to_string(result.op_kind); delim = "_";
|
||||
out << delim << result.operation_name;
|
||||
|
||||
for (auto const & arg : result.arguments) {
|
||||
out << delim << arg.second;
|
||||
}
|
||||
|
||||
out << "\" ";
|
||||
|
||||
bool skipped = false, failed = false, error = false;
|
||||
|
||||
switch (result.disposition) {
|
||||
case Disposition::kNotRun:
|
||||
case Disposition::kNotSupported:
|
||||
skipped = true;
|
||||
break;
|
||||
case Disposition::kPassed:
|
||||
case Disposition::kNotVerified:
|
||||
break;
|
||||
case Disposition::kFailed:
|
||||
case Disposition::kIncorrect:
|
||||
failed = true;
|
||||
break;
|
||||
case Disposition::kInvalidProblem:
|
||||
case Disposition::kInvalid:
|
||||
error = true;
|
||||
break;
|
||||
};
|
||||
|
||||
if (skipped) {
|
||||
out << "status=\"notrun\"";
|
||||
} else {
|
||||
out << "status=\"run\"";
|
||||
}
|
||||
|
||||
out << ">" << std::endl;
|
||||
|
||||
if (failed) {
|
||||
out << " <failure message=\"" << to_string(result.disposition) << "\" />" << std::endl;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
out << " <error message=\"" << to_string(result.disposition) << "\" />" << std::endl;
|
||||
}
|
||||
|
||||
out << " <system-out><![CDATA[" << std::endl;
|
||||
std::stringstream ss;
|
||||
print_result_pretty_(ss, result, false);
|
||||
out << escape_xml_special_chars(ss.str()) << std::endl;
|
||||
out << " ]]></system-out>" << std::endl;
|
||||
|
||||
out << " </testcase>" << std::endl;
|
||||
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
std::ostream & PerformanceReport::print_junit_footer_(std::ostream &out) {
|
||||
|
||||
out << "</testsuite>" << std::endl;
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
|
||||
@@ -59,6 +59,12 @@ private:
|
||||
/// Output file containing results
|
||||
std::ofstream output_file_;
|
||||
|
||||
/// Operation file name containing junit performance report of op_kind
|
||||
std::string op_junit_file_name_;
|
||||
|
||||
/// Output file containing junit results
|
||||
std::ofstream junit_output_file_;
|
||||
|
||||
/// Flag indicating the performance report is valid
|
||||
bool good_;
|
||||
|
||||
@@ -74,6 +80,7 @@ private:
|
||||
public:
|
||||
|
||||
PerformanceReport(Options const &options, std::vector<std::string> const &argument_names, library::OperationKind const &op_kind);
|
||||
~PerformanceReport();
|
||||
|
||||
bool good() const { return good_; }
|
||||
|
||||
@@ -81,8 +88,6 @@ public:
|
||||
void append_result(PerformanceResult result);
|
||||
void append_results(PerformanceResultVector const &results);
|
||||
|
||||
void close();
|
||||
|
||||
public:
|
||||
|
||||
/// Prints the CSV header
|
||||
@@ -91,10 +96,21 @@ public:
|
||||
/// Prints the CSV
|
||||
std::ostream & print_result_csv_(std::ostream &out, PerformanceResult const &result);
|
||||
|
||||
/// @defgroup jUnit Result Generation
|
||||
/// Functions related to generation of the jUnit results
|
||||
/// @{
|
||||
|
||||
std::ostream & print_junit_header_(std::ostream &out);
|
||||
std::ostream & print_junit_result_(std::ostream &out, PerformanceResult const &result);
|
||||
std::ostream & print_junit_footer_(std::ostream &out);
|
||||
|
||||
/// @}
|
||||
|
||||
/// Prints the result in human readable form
|
||||
std::ostream & print_result_pretty_(
|
||||
std::ostream &out,
|
||||
PerformanceResult const &result);
|
||||
PerformanceResult const &result,
|
||||
bool use_shell_coloring = true);
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -961,6 +961,85 @@ bool arg_as_SplitKModeID(
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
KernelArgument::Value const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
|
||||
conv_mode = library::from_string<library::ConvModeID>(
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
|
||||
|
||||
if (conv_mode == library::ConvModeID::kInvalid) {
|
||||
throw std::runtime_error(
|
||||
"arg_as_ConvModeID() - illegal cast.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
||||
throw std::runtime_error(
|
||||
"arg_as_ConvModeID() - illegal cast.");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
return arg_as_ConvModeID(conv_mode, value_ptr);
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
KernelArgument::Value const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
|
||||
provider = library::from_string<library::Provider>(
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr)->element);
|
||||
|
||||
if (provider == library::Provider::kInvalid) {
|
||||
throw std::runtime_error(
|
||||
"arg_as_ProviderID() - illegal cast.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
||||
throw std::runtime_error(
|
||||
"arg_as_ProviderID() - illegal cast.");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
return arg_as_ProviderID(provider, value_ptr);
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
|
||||
bool arg_as_scalar(
|
||||
std::vector<uint8_t> &bytes,
|
||||
@@ -1049,9 +1128,94 @@ bool tensor_description_satisfies(
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns true if conv_kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
library::ConvKind conv_kind_cmd_line =
|
||||
library::from_string<library::ConvKind>(value_ptr->element);
|
||||
|
||||
if (conv_kind_cmd_line != library::ConvKind::kUnknown &&
|
||||
conv_kind_cmd_line != conv_kind) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns true if conv_kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
return conv_kind_satisfies(
|
||||
conv_kind,
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr));
|
||||
}
|
||||
else {
|
||||
throw std::runtime_error("Kernel argument mismatch");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr) {
|
||||
|
||||
if (value_ptr->not_null) {
|
||||
library::IteratorAlgorithmID iterator_algorithm_cmd_line =
|
||||
library::from_string<library::IteratorAlgorithmID>(value_ptr->element);
|
||||
|
||||
if (iterator_algorithm_cmd_line != library::IteratorAlgorithmID::kNone &&
|
||||
iterator_algorithm_cmd_line != iterator_algorithm) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem) {
|
||||
|
||||
size_t idx = problem_space.argument_index(name);
|
||||
KernelArgument::Value const *value_ptr = problem.at(idx).get();
|
||||
|
||||
if (value_ptr->argument->description->type == ArgumentTypeID::kEnumerated) {
|
||||
return iterator_algorithm_satisfies(
|
||||
iterator_algorithm,
|
||||
static_cast<EnumeratedTypeArgument::EnumeratedTypeValue const *>(value_ptr));
|
||||
}
|
||||
else {
|
||||
throw std::runtime_error("Kernel argument mismatch");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -909,6 +909,37 @@ bool arg_as_SplitKModeID(
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(library::ConvModeID &conv_mode, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ConvModeID(
|
||||
library::ConvModeID &conv_mode,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_IteratorAlgorithmID(library::IteratorAlgorithmID &iterator_algorithm, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_IteratorAlgorithmID(
|
||||
library::IteratorAlgorithmID &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(library::Provider &provider, KernelArgument::Value const *value_ptr);
|
||||
|
||||
/// Lexically casts an argument to an int64 if it is defined. Returns true if not null.
|
||||
bool arg_as_ProviderID(
|
||||
library::Provider &provider,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Lexically casts an argument to a given type stored in a byte array. Returns true if not null.
|
||||
bool arg_as_scalar(
|
||||
std::vector<uint8_t> &bytes,
|
||||
@@ -935,10 +966,34 @@ bool tensor_description_satisfies(
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
|
||||
/// Returns true if a conv kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr);
|
||||
|
||||
/// Returns true if a conv kind satisfies the value
|
||||
bool conv_kind_satisfies(
|
||||
library::ConvKind const &conv_kind,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
EnumeratedTypeArgument::EnumeratedTypeValue const *value_ptr);
|
||||
|
||||
/// Returns true if a iterator algorithm satisfies the value
|
||||
bool iterator_algorithm_satisfies(
|
||||
library::IteratorAlgorithmID const &iterator_algorithm,
|
||||
char const *name,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines profiling functionality for reduction operation
|
||||
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
// CUTLASS Library includes
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
// Profiler includes
|
||||
#include "options.h"
|
||||
#include "device_context.h"
|
||||
#include "operation_profiler.h"
|
||||
#include "performance_result.h"
|
||||
#include "problem_space.h"
|
||||
#if CUTLASS_ENABLE_CUDNN
|
||||
#include "cudnn_helpers.h"
|
||||
#endif //#if CUTLASS_ENABLE_CUDNN
|
||||
#include "debug.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace profiler {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Abstract base class for each math function
|
||||
class ReductionOperationProfiler : public OperationProfiler {
|
||||
public:
|
||||
|
||||
|
||||
/// Workspace used
|
||||
struct ReductionWorkspace {
|
||||
|
||||
/// Conv device allocations
|
||||
DeviceAllocation *Workspace;
|
||||
DeviceAllocation *Source;
|
||||
DeviceAllocation *Destination;
|
||||
DeviceAllocation *Reference;
|
||||
|
||||
/// Library configuration and arguments
|
||||
library::ReductionConfiguration configuration;
|
||||
library::ReductionArguments arguments;
|
||||
|
||||
/// Buffer used for the cutlass operations' host workspace
|
||||
std::vector<uint8_t> host_workspace;
|
||||
|
||||
/// Buffer used for the cutlass operations' device workspace
|
||||
DeviceAllocation device_workspace;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
ReductionWorkspace():
|
||||
Workspace(nullptr), Source(nullptr), Destination(nullptr), Reference(nullptr) { }
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Reduction problem obtained from problem space
|
||||
MatrixCoord problem_;
|
||||
|
||||
/// Device memory allocations
|
||||
ReductionWorkspace conv_workspace_;
|
||||
|
||||
|
||||
public:
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
ReductionOperationProfiler(Options const &options);
|
||||
|
||||
/// Destructor
|
||||
virtual ~ReductionOperationProfiler();
|
||||
|
||||
/// Prints usage statement for the math function
|
||||
virtual void print_usage(std::ostream &out) const;
|
||||
|
||||
/// Prints examples
|
||||
virtual void print_examples(std::ostream &out) const;
|
||||
|
||||
/// Extracts the problem dimensions
|
||||
virtual Status initialize_configuration(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Initializes workspace
|
||||
virtual Status initialize_workspace(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Verifies CUTLASS against references
|
||||
virtual bool verify_cutlass(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
/// Measures performance results
|
||||
virtual bool profile(
|
||||
Options const &options,
|
||||
PerformanceReport &report,
|
||||
DeviceContext &device_context,
|
||||
library::Operation const *operation,
|
||||
ProblemSpace const &problem_space,
|
||||
ProblemSpace::Problem const &problem);
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace profiler
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -227,6 +227,9 @@ void SparseGemmOperationProfiler::SparseGemmProblem::initialize_result(
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "E", problem_space,
|
||||
std::string(library::to_string(operation_desc.E.element)) + ":" + library::to_string(operation_desc.E.layout));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
|
||||
@@ -62,6 +62,18 @@ void reorder_column(TensorRef<Element, Layout> dest,
|
||||
}
|
||||
}
|
||||
|
||||
template <int Interleaved, typename Element, typename Layout>
|
||||
void reorder_convK(TensorRef<Element, Layout> dest,
|
||||
TensorRef<Element, Layout> src,
|
||||
cutlass::gemm::GemmCoord problem_size) {
|
||||
|
||||
TensorRef<Element, layout::RowMajorInterleaved<Interleaved>> mappedDest(dest.data(), dest.stride(0));
|
||||
TensorRef<Element, layout::RowMajorInterleaved<Interleaved>> mappedSrc(src.data(), src.stride(0));
|
||||
|
||||
reorder_column<Interleaved>(
|
||||
mappedDest, mappedSrc, problem_size);
|
||||
}
|
||||
|
||||
/// This is needed for the sparse tensor core kernels. The purpose
|
||||
/// is to use ldmatrix to load from shared memory to the register file.
|
||||
template <typename Element, typename LayoutDest, typename LayoutSrc>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,767 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Reference implementation for convolution in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/functional.h"
|
||||
#include "cutlass/layout/tensor.h"
|
||||
#include "cutlass/numeric_conversion.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
#include "cutlass/tensor_ref.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/conv/convolution.h"
|
||||
#include "cutlass/conv/conv2d_problem_size.h"
|
||||
#include "cutlass/conv/conv3d_problem_size.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace host {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Forward propagation
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// y = conv2d(x, w)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv2dFprop(
|
||||
conv::Conv2dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_x,
|
||||
TensorRef<ElementB, LayoutB> tensor_w,
|
||||
TensorRef<ElementC, LayoutC> tensor_y_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_y_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
ConvertOp convert_op;
|
||||
InnerProductOp inner_product_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int p = 0; p < problem_size.P; ++p) {
|
||||
for (int q = 0; q < problem_size.Q; ++q) {
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
int h = p * problem_size.stride_h - problem_size.pad_h + filter_r * problem_size.dilation_h;
|
||||
int w = q * problem_size.stride_w - problem_size.pad_w + filter_s * problem_size.dilation_w;
|
||||
|
||||
if (h >= 0 && h < problem_size.H && w >= 0 && w < problem_size.W) {
|
||||
|
||||
ElementA a = tensor_x.at({n, h, w, c});
|
||||
ElementB b = tensor_w.at({k, r, s, c});
|
||||
|
||||
acc = inner_product_op(ElementAccumulator(a), ElementAccumulator(b), acc);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_y_in.at(cutlass::make_Coord(n, p, q, k));
|
||||
}
|
||||
|
||||
tensor_y_out.at(cutlass::make_Coord(n, p, q, k)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Depthwise-separable convolution
|
||||
template <typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementAccumulator,
|
||||
typename ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator> >
|
||||
void Depsep_Fprop(
|
||||
cutlass::TensorView<ElementA, LayoutA> tensor_A,
|
||||
cutlass::TensorView<ElementB, LayoutB> tensor_B,
|
||||
cutlass::TensorView<ElementC, LayoutC> tensor_C,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta,
|
||||
cutlass::Tensor4DCoord padding,
|
||||
cutlass::Coord<2> conv_stride,
|
||||
cutlass::Coord<2> dilation,
|
||||
cutlass::conv::Mode mode = cutlass::conv::Mode::kCrossCorrelation) {
|
||||
|
||||
ConvertOp convert_op;
|
||||
InnerProductOp inner_product_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int n = 0; n < tensor_C.extent().n(); ++n) {
|
||||
for (int p = 0; p < tensor_C.extent().h(); ++p) {
|
||||
for (int q = 0; q < tensor_C.extent().w(); ++q) {
|
||||
for (int g = 0; g < tensor_C.extent().c(); ++g) {
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
for (int r = 0; r < tensor_B.extent().h(); ++r) {
|
||||
for (int s = 0; s < tensor_B.extent().w(); ++s) {
|
||||
if ((p * conv_stride[0] - padding[0] + r * dilation[0]) < tensor_A.extent().h() &&
|
||||
(p * conv_stride[0] - padding[0] + r * dilation[0]) >= 0 &&
|
||||
(q * conv_stride[1] - padding[2] + s * dilation[1]) < tensor_A.extent().w() &&
|
||||
(q * conv_stride[1] - padding[2] + s * dilation[1]) >= 0) {
|
||||
ElementA a = tensor_A.at(
|
||||
cutlass::make_Coord(n,
|
||||
p * conv_stride[0] - padding[0] + r * dilation[0],
|
||||
q * conv_stride[1] - padding[2] + s * dilation[1],
|
||||
g));
|
||||
|
||||
ElementB b = (mode == cutlass::conv::Mode::kCrossCorrelation)
|
||||
? tensor_B.at(cutlass::make_Coord(g, r, s, 0))
|
||||
: tensor_B.at(cutlass::make_Coord(
|
||||
g, tensor_B.extent().h() - r - 1, tensor_B.extent().w() - s - 1, 0));
|
||||
|
||||
acc = inner_product_op(ElementAccumulator(a), ElementAccumulator(b), acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = tensor_C.at(cutlass::make_Coord(n, p, q, g));
|
||||
tensor_C.at(cutlass::make_Coord(n, p, q, g)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Dgrad
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// dx = dgrad(dy, w)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv2dDgrad(
|
||||
cutlass::conv::Conv2dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_dy,
|
||||
TensorRef<ElementB, LayoutB> tensor_w,
|
||||
TensorRef<ElementC, LayoutC> tensor_dx_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_dx_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
ConvertOp convert_op;
|
||||
InnerProductOp inner_product_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int h = 0; h < problem_size.H; ++h) {
|
||||
for (int w = 0; w < problem_size.W; ++w) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
int p = h + problem_size.pad_h - filter_r * problem_size.dilation_h;
|
||||
int q = w + problem_size.pad_w - filter_s * problem_size.dilation_w;
|
||||
|
||||
if (p >= 0 && (p % problem_size.stride_h) == 0 &&
|
||||
q >= 0 && (q % problem_size.stride_w) == 0) {
|
||||
|
||||
p = p / problem_size.stride_h;
|
||||
q = q / problem_size.stride_w;
|
||||
|
||||
if (p < problem_size.P && q < problem_size.Q) {
|
||||
|
||||
ElementA a = tensor_dy.at(cutlass::make_Coord(n, p, q, k));
|
||||
ElementB b = tensor_w.at(cutlass::make_Coord(k, r, s, c));
|
||||
|
||||
acc = inner_product_op(ElementAccumulator(a), ElementAccumulator(b), acc);
|
||||
}
|
||||
}
|
||||
|
||||
} // for (K)
|
||||
} // for (S)
|
||||
} // for (R)
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_dx_in.at(cutlass::make_Coord(n, h, w, c));
|
||||
}
|
||||
|
||||
tensor_dx_out.at(cutlass::make_Coord(n, h, w, c)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
|
||||
} // for (C)
|
||||
} // for (W)
|
||||
} // for (H)
|
||||
} // for (N)
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Wgrad
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// dw = wgrad(dy, x)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv2dWgrad(
|
||||
cutlass::conv::Conv2dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_dy,
|
||||
TensorRef<ElementB, LayoutB> tensor_x,
|
||||
TensorRef<ElementC, LayoutC> tensor_dw_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_dw_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
InnerProductOp inner_product_op;
|
||||
ConvertOp convert_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int p = 0; p < problem_size.P; ++p) {
|
||||
for (int q = 0; q < problem_size.Q; ++q) {
|
||||
|
||||
cutlass::Tensor4DCoord b_coord;
|
||||
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
b_coord = make_Coord(
|
||||
n,
|
||||
p * problem_size.stride_h - problem_size.pad_h + filter_r * problem_size.dilation_h,
|
||||
q * problem_size.stride_w - problem_size.pad_w + filter_s * problem_size.dilation_w,
|
||||
c);
|
||||
|
||||
if (b_coord.h() < problem_size.H && b_coord.h() >= 0 &&
|
||||
b_coord.w() < problem_size.W && b_coord.w() >= 0) {
|
||||
|
||||
ElementAccumulator a = ElementAccumulator(tensor_dy.at(cutlass::make_Coord(n, p, q, k)));
|
||||
ElementAccumulator b = ElementAccumulator(tensor_x.at(b_coord));
|
||||
acc = inner_product_op(a, b, acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_dw_in.at(cutlass::make_Coord(k, r, s, c));
|
||||
}
|
||||
|
||||
tensor_dw_out.at(cutlass::make_Coord(k, r, s, c)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
|
||||
} // for (C)
|
||||
} // for (S)
|
||||
} // for (R)
|
||||
} // for (K)
|
||||
}
|
||||
|
||||
/// Generic 2D convolution targeting Conv2dFprop, Conv2dDgrad, and Conv2dWgrad.
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv2d(
|
||||
conv::Operator convolutional_operator,
|
||||
conv::Conv2dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_A,
|
||||
TensorRef<ElementB, LayoutB> tensor_B,
|
||||
TensorRef<ElementC, LayoutC> tensor_C,
|
||||
TensorRef<ElementC, LayoutC> tensor_D,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
switch (convolutional_operator) {
|
||||
case conv::Operator::kFprop:
|
||||
Conv2dFprop<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
case conv::Operator::kDgrad:
|
||||
Conv2dDgrad<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
case conv::Operator::kWgrad:
|
||||
Conv2dWgrad<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// 3D convolution
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// y = conv3d(x, w)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv3dFprop(
|
||||
conv::Conv3dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_x,
|
||||
TensorRef<ElementB, LayoutB> tensor_w,
|
||||
TensorRef<ElementC, LayoutC> tensor_y_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_y_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
ConvertOp convert_op;
|
||||
InnerProductOp inner_product_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int z = 0; z < problem_size.Z; ++z) {
|
||||
for (int p = 0; p < problem_size.P; ++p) {
|
||||
for (int q = 0; q < problem_size.Q; ++q) {
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int t = 0; t < problem_size.T; ++t) {
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
int filter_t = t;
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_t = problem_size.T - 1 - t;
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
int d = z * problem_size.stride_d - problem_size.pad_d + filter_t * problem_size.dilation_d;
|
||||
int h = p * problem_size.stride_h - problem_size.pad_h + filter_r * problem_size.dilation_h;
|
||||
int w = q * problem_size.stride_w - problem_size.pad_w + filter_s * problem_size.dilation_w;
|
||||
|
||||
if (d >= 0 && d < problem_size.D &&
|
||||
h >=0 && h < problem_size.H &&
|
||||
w >= 0 && w < problem_size.W) {
|
||||
|
||||
ElementA a = tensor_x.at({n, d, h, w, c});
|
||||
ElementB b = tensor_w.at({k, t, r, s, c});
|
||||
|
||||
acc = inner_product_op(ElementAccumulator(a), ElementAccumulator(b), acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_y_in.at(cutlass::make_Coord(n, z, p, q, k));
|
||||
}
|
||||
|
||||
tensor_y_out.at(cutlass::make_Coord(n, z, p, q, k)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Dgrad
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// dx = dgrad(dy, w)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv3dDgrad(
|
||||
cutlass::conv::Conv3dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_dy,
|
||||
TensorRef<ElementB, LayoutB> tensor_w,
|
||||
TensorRef<ElementC, LayoutC> tensor_dx_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_dx_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
ConvertOp convert_op;
|
||||
InnerProductOp inner_product_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int d = 0; d < problem_size.D; ++d) {
|
||||
for (int h = 0; h < problem_size.H; ++h) {
|
||||
for (int w = 0; w < problem_size.W; ++w) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int t = 0; t < problem_size.T; ++t) {
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
|
||||
int filter_t = t;
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_t = problem_size.T - 1 - t;
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
int z = d + problem_size.pad_d - filter_t * problem_size.dilation_d;
|
||||
int p = h + problem_size.pad_h - filter_r * problem_size.dilation_h;
|
||||
int q = w + problem_size.pad_w - filter_s * problem_size.dilation_w;
|
||||
|
||||
if (z >= 0 && (z % problem_size.stride_d) == 0 &&
|
||||
p >= 0 && (p % problem_size.stride_h) == 0 &&
|
||||
q >= 0 && (q % problem_size.stride_w) == 0) {
|
||||
|
||||
z = z / problem_size.stride_d;
|
||||
p = p / problem_size.stride_h;
|
||||
q = q / problem_size.stride_w;
|
||||
|
||||
if (z < problem_size.Z && p < problem_size.P && q < problem_size.Q) {
|
||||
|
||||
ElementA a = tensor_dy.at(cutlass::make_Coord(n, z, p, q, k));
|
||||
ElementB b = tensor_w.at(cutlass::make_Coord(k, t, r, s, c));
|
||||
|
||||
acc = inner_product_op(ElementAccumulator(a), ElementAccumulator(b), acc);
|
||||
}
|
||||
}
|
||||
|
||||
} // for (K)
|
||||
} // for (S)
|
||||
} // for (R)
|
||||
} // for (T)
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_dx_in.at(cutlass::make_Coord(n, d, h, w, c));
|
||||
}
|
||||
|
||||
tensor_dx_out.at(cutlass::make_Coord(n, d, h, w, c)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
|
||||
} // for (C)
|
||||
} // for (W)
|
||||
} // for (H)
|
||||
} // for (D)
|
||||
} // for (N)
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Wgrad
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// dw = wgrad(dy, x)
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv3dWgrad(
|
||||
cutlass::conv::Conv3dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_dy,
|
||||
TensorRef<ElementB, LayoutB> tensor_x,
|
||||
TensorRef<ElementC, LayoutC> tensor_dw_in,
|
||||
TensorRef<ElementC, LayoutC> tensor_dw_out,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
InnerProductOp inner_product_op;
|
||||
ConvertOp convert_op;
|
||||
|
||||
// Apply MMA and accumulate ElementAccumulator
|
||||
for (int k = 0; k < problem_size.K; ++k) {
|
||||
for (int t = 0; t < problem_size.T; ++t) {
|
||||
for (int r = 0; r < problem_size.R; ++r) {
|
||||
for (int s = 0; s < problem_size.S; ++s) {
|
||||
for (int c = 0; c < problem_size.C; ++c) {
|
||||
|
||||
ElementAccumulator acc = ElementAccumulator();
|
||||
|
||||
for (int n = 0; n < problem_size.N; ++n) {
|
||||
for (int z = 0; z < problem_size.Z; ++z) {
|
||||
for (int p = 0; p < problem_size.P; ++p) {
|
||||
for (int q = 0; q < problem_size.Q; ++q) {
|
||||
|
||||
int filter_t = t;
|
||||
int filter_r = r;
|
||||
int filter_s = s;
|
||||
|
||||
if (problem_size.mode == cutlass::conv::Mode::kConvolution) {
|
||||
filter_t = problem_size.T - 1 - t;
|
||||
filter_r = problem_size.R - 1 - r;
|
||||
filter_s = problem_size.S - 1 - s;
|
||||
}
|
||||
|
||||
Tensor5DCoord b_coord = make_Coord(
|
||||
n,
|
||||
z * problem_size.stride_d - problem_size.pad_d + filter_t * problem_size.dilation_d,
|
||||
p * problem_size.stride_h - problem_size.pad_h + filter_r * problem_size.dilation_h,
|
||||
q * problem_size.stride_w - problem_size.pad_w + filter_s * problem_size.dilation_w,
|
||||
c);
|
||||
|
||||
if (b_coord.d() < problem_size.D && b_coord.d() >= 0 &&
|
||||
b_coord.h() < problem_size.H && b_coord.h() >= 0 &&
|
||||
b_coord.w() < problem_size.W && b_coord.w() >= 0) {
|
||||
|
||||
ElementAccumulator a = ElementAccumulator(tensor_dy.at(cutlass::make_Coord(n, z, p, q, k)));
|
||||
ElementAccumulator b = ElementAccumulator(tensor_x.at(b_coord));
|
||||
|
||||
acc = inner_product_op(a, b, acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply Epilogue, compute ElementCompute, convert and store ElementC
|
||||
ElementC c_ref = ElementC();
|
||||
|
||||
if (beta != ElementCompute()) {
|
||||
c_ref = tensor_dw_in.at(cutlass::make_Coord(k, t, r, s, c));
|
||||
}
|
||||
|
||||
tensor_dw_out.at(cutlass::make_Coord(k, t, r, s, c)) =
|
||||
convert_op(alpha * ElementCompute(acc) + beta * ElementCompute(c_ref));
|
||||
|
||||
} // for (C)
|
||||
} // for (S)
|
||||
} // for (R)
|
||||
} // for (T)
|
||||
} // for (K)
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Generic 3D convolution targeting Conv2dFprop, Conv2dDgrad, and Conv2dWgrad.
|
||||
template <
|
||||
typename ElementA,
|
||||
typename LayoutA,
|
||||
typename ElementB,
|
||||
typename LayoutB,
|
||||
typename ElementC,
|
||||
typename LayoutC,
|
||||
typename ElementCompute,
|
||||
typename ElementAccumulator = ElementCompute,
|
||||
typename ConvertOp = NumericConverter<ElementC, ElementCompute>,
|
||||
typename InnerProductOp = multiply_add<ElementAccumulator>
|
||||
>
|
||||
void Conv3d(
|
||||
conv::Operator convolutional_operator,
|
||||
conv::Conv3dProblemSize problem_size,
|
||||
TensorRef<ElementA, LayoutA> tensor_A,
|
||||
TensorRef<ElementB, LayoutB> tensor_B,
|
||||
TensorRef<ElementC, LayoutC> tensor_C,
|
||||
TensorRef<ElementC, LayoutC> tensor_D,
|
||||
ElementCompute alpha,
|
||||
ElementCompute beta) {
|
||||
|
||||
switch (convolutional_operator) {
|
||||
case conv::Operator::kFprop:
|
||||
Conv3dFprop<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
case conv::Operator::kDgrad:
|
||||
Conv3dDgrad<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
case conv::Operator::kWgrad:
|
||||
Conv3dWgrad<
|
||||
ElementA, LayoutA,
|
||||
ElementB, LayoutB,
|
||||
ElementC, LayoutC,
|
||||
ElementCompute,
|
||||
ElementAccumulator,
|
||||
ConvertOp, InnerProductOp
|
||||
>(problem_size, tensor_A, tensor_B, tensor_C, tensor_D, alpha, beta);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace host
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -249,6 +249,45 @@ struct Gemm<ElementA, LayoutA, ElementB, LayoutB, ElementC, LayoutC, ScalarType,
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Partial specialization for multiply-add
|
||||
template <typename ElementA, typename LayoutA, typename ElementB,
|
||||
typename LayoutB, typename ElementC, typename LayoutC,
|
||||
typename ScalarType, typename ComputeType>
|
||||
struct Gemm<ElementA, LayoutA, ElementB, LayoutB, ElementC, LayoutC, ScalarType,
|
||||
ComputeType, arch::OpMultiplyAddFastBF16> {
|
||||
|
||||
void operator()(gemm::GemmCoord problem_size, ScalarType alpha,
|
||||
TensorRef<ElementA, LayoutA> tensor_a,
|
||||
TensorRef<ElementB, LayoutB> tensor_b, ScalarType beta,
|
||||
TensorRef<ElementC, LayoutC> tensor_c,
|
||||
ComputeType initial_accum = ComputeType(0)) {
|
||||
static_assert(
|
||||
LayoutA::kRank == 2 && LayoutB::kRank == 2 && LayoutC::kRank == 2,
|
||||
"Tensors must be of rank 2");
|
||||
|
||||
compute_gemm<ElementA, LayoutA, ElementB, LayoutB, ElementC, LayoutC,
|
||||
ScalarType, ComputeType, multiply_add<ComputeType>>(
|
||||
problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, initial_accum);
|
||||
}
|
||||
|
||||
void operator()(gemm::GemmCoord problem_size, ScalarType alpha,
|
||||
TensorRef<ElementA, LayoutA> tensor_a,
|
||||
TensorRef<ElementB, LayoutB> tensor_b, ScalarType beta,
|
||||
TensorRef<ElementC, LayoutC> tensor_c,
|
||||
TensorRef<ElementC, LayoutC> tensor_d,
|
||||
ComputeType initial_accum = ComputeType(0)) {
|
||||
static_assert(
|
||||
LayoutA::kRank == 2 && LayoutB::kRank == 2 && LayoutC::kRank == 2,
|
||||
"Tensors must be of rank 2");
|
||||
|
||||
compute_gemm<ElementA, LayoutA, ElementB, LayoutB, ElementC, LayoutC,
|
||||
ScalarType, ComputeType, multiply_add<ComputeType>>(
|
||||
problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, tensor_d, initial_accum);
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Partial specialization for multiply-add-saturate
|
||||
template <typename ElementA, typename LayoutA, typename ElementB,
|
||||
typename LayoutB, typename ElementC, typename LayoutC,
|
||||
|
||||
Reference in New Issue
Block a user