leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Updates to fused epilogue (#383)

Browse Source 
* Enhancements and fixes to fused GEMM and Convolution epilogue.
* Need to explicitly list cudart as unit test library dependency.
This commit is contained in:
Andrew Kerr
2021-12-17 16:04:43 -05:00
committed by GitHub
parent 4e666e1dfd
commit ec4f7e5194
24 changed files with 372 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/cutlass/arch/wmma_sm75.h
View File

@@ -109,6 +109,7 @@ struct Wmma<
FragmentB const &B,
FragmentC const &C) const {
nvcuda::wmma::mma_sync(D, A, B, C);
}
#else
@@ -186,7 +187,6 @@ struct Wmma<
FragmentA const &A,
FragmentB const &B,
FragmentC const &C) const {
nvcuda::wmma::bmma_sync(D, A, B, C, nvcuda::wmma::experimental::bmmaBitOpXOR,
nvcuda::wmma::experimental::bmmaAccumulateOpPOPC);
}

6
include/cutlass/cutlass.h
View File

@@ -109,6 +109,12 @@ static char const* cutlassGetStatusString(cutlass::Status status) {
////////////////////////////////////////////////////////////////////////////////////////////////////
#ifndef CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED
#define CUTLASS_CONV_UNIT_TEST_RIGOROUS_SIZE_ENABLED 0
#endif
// CUDA 10.1 introduces the mma instruction
#if !defined(CUTLASS_ENABLE_TENSOR_CORE_MMA)
#define CUTLASS_ENABLE_TENSOR_CORE_MMA 0

2
include/cutlass/epilogue/thread/activation.h
View File

@@ -58,6 +58,7 @@ struct Identity {
/// ReLu operator - propagates NaNs
template <typename T>
struct ReLu {
static const bool kIsHeavy=false;
CUTLASS_HOST_DEVICE
T operator()(T const & threshold, T value) const {
if (value < threshold) {
@@ -76,6 +77,7 @@ struct ReLu {
template <typename T, int N>
struct ReLu<Array<T, N>> {
static const bool kIsHeavy=false;
CUTLASS_HOST_DEVICE
Array<T, N> operator()(T const & threshold, Array<T, N> const &frag) const {
Array<T, N> result;

8
include/cutlass/epilogue/thread/linear_combination_bias_elementwise.h
View File

@@ -201,8 +201,8 @@ public:
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < kElementsPerAccess; ++i) {
ElementCompute z = binary_op(alpha_ * tmp_Accum[i] + beta_ * tmp_C[i], V[i]);
result_Z[i] = z;
result_T[i] = skip_elementwise_ ? z : elementwise_op(z);
result_T[i] = z;
result_Z[i] = skip_elementwise_ ? z : elementwise_op(z);
}
NumericArrayConverter<ElementZ, ElementCompute, kElementsPerAccess> convert_z;
@@ -230,8 +230,8 @@ public:
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < kElementsPerAccess; ++i) {
ElementCompute z = binary_op(alpha_ * tmp_Accum[i], V[i]);
result_Z[i] = z;
result_T[i] = skip_elementwise_ ? z : elementwise_op(z);
result_T[i] = z;
result_Z[i] = skip_elementwise_ ? z : elementwise_op(z);
}
NumericArrayConverter<ElementZ, ElementCompute, kElementsPerAccess> convert_z;

2
include/cutlass/epilogue/threadblock/epilogue_with_broadcast.h
View File

@@ -306,6 +306,7 @@ public:
/// Debug printing
CUTLASS_DEVICE
static void print() {
#if 0
printf("BroadcastDetail {\n");
printf(
" kColumnsPerThread: %d\nkRowsPerThread: %d\n,kThreadCount: %d\nkThreadsPerRow: %d\n"
@@ -321,6 +322,7 @@ public:
StorageShape::kCount
);
printf("};\n");
#endif
}
};

2
include/cutlass/epilogue/threadblock/epilogue_with_reduction.h
View File

@@ -212,6 +212,7 @@ public:
/// Debug printing
CUTLASS_DEVICE
static void print() {
#if 0
printf("ReductionDetail {\n");
printf(
" kElementsPerAccess:%d\nkColumnsPerThread: %d\nkRowsPerThread: %d\n,kThreadCount: %d\nkThreadsPerRow: %d\n"
@@ -228,6 +229,7 @@ public:
StorageShape::kCount
);
printf("};\n");
#endif
}
};

9
include/cutlass/tfloat32.h
View File

@@ -363,8 +363,13 @@ tfloat32_t operator+(tfloat32_t const& lhs, tfloat32_t const& rhs) {
CUTLASS_HOST_DEVICE
tfloat32_t operator-(tfloat32_t const& lhs) {
float x = -reinterpret_cast<float const &>(lhs);
return *reinterpret_cast<tfloat32_t *>(&x);
union u_tff32 {
float val_f32;
tfloat32_t val_tf;
CUTLASS_HOST_DEVICE u_tff32() : val_f32(0) { }
};
union u_tff32 x; x.val_f32 = -reinterpret_cast<float const &>(lhs);
return x.val_tf;
}
CUTLASS_HOST_DEVICE