leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 2.4 (Implicit GEMM convolution) (#147)

Browse Source 
CUTLASS 2.4 (Implicit GEMM Convolution)

Co-authored-by: Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
This commit is contained in:
Manish Gupta
2020-11-19 21:25:25 -08:00
committed by GitHub
co-authored by Manish Gupta <manigupta@nvidia.com>, Haicheng Wu <haichengw@nvidia.com>, Dustyn Blasig <dblasig@nvidia.com>, Andrew Kerr <akerr@nvidia.com>
parent c2b80ad4e4
commit 6615010cd0
224 changed files with 43939 additions and 1061 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/cutlass/arch/memory_sm80.h
+16 -1
View File
@@ -74,6 +74,10 @@ template <
/// Size of the access in bytes
int SizeInBytes>
struct cp_async<SizeInBytes, CacheOperation::Always> {
// Make sure the size is supported.
static_assert((SizeInBytes == 4 || SizeInBytes == 8 || SizeInBytes == 16),
"Size is not supported");
/// Copy
CUTLASS_DEVICE
cp_async(void *smem_ptr, void const *global_ptr, bool pred_guard = true) {
@@ -104,6 +108,10 @@ template <
/// Size of the access in bytes
int SizeInBytes>
struct cp_async_zfill<SizeInBytes, CacheOperation::Always> {
// Make sure the size is supported.
static_assert((SizeInBytes == 4 || SizeInBytes == 8 || SizeInBytes == 16),
"Size is not supported");
/// Copy with zero fill
CUTLASS_DEVICE
cp_async_zfill(void *smem_ptr, void const *global_ptr, bool pred_guard) {
@@ -138,6 +146,10 @@ template <
/// Size of the access in bytes
int SizeInBytes>
struct cp_async<SizeInBytes, CacheOperation::Global> {
// Make sure the size is supported.
static_assert((SizeInBytes == 4 || SizeInBytes == 8 || SizeInBytes == 16),
"Size is not supported");
/// Copy
CUTLASS_DEVICE
cp_async(void *smem_ptr, void const *global_ptr, bool pred_guard = true) {
@@ -171,6 +183,10 @@ template <
/// Size of the access in bytes
int SizeInBytes>
struct cp_async_zfill<SizeInBytes, CacheOperation::Global> {
// Make sure the size is supported.
static_assert((SizeInBytes == 4 || SizeInBytes == 8 || SizeInBytes == 16),
"Size is not supported");
/// Copy with zero fill
CUTLASS_DEVICE
cp_async_zfill(void *smem_ptr, void const *global_ptr, bool pred_guard = true) {
@@ -235,4 +251,3 @@ CUTLASS_DEVICE void cp_async_wait<0>() {
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
include/cutlass/arch/mma.h
+1 -1
View File
@@ -201,5 +201,5 @@ struct SparseMma;
#include "cutlass/arch/mma_sm70.h"
#include "cutlass/arch/mma_sm75.h"
#include "cutlass/arch/mma_sm80.h"
#include "cutlass/arch/sp_mma_sm80.h"
#include "cutlass/arch/mma_sparse_sm80.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
include/cutlass/arch/mma_sm75.h
+2 -2
View File
@@ -365,7 +365,7 @@ struct Mma<
}
};
/// Matrix multiply-add operation: S32 = S8 * U8 + S32
/// Matrix multiply-add operation: S32 = U8 * U8 + S32
template <>
struct Mma<
gemm::GemmShape<8, 8, 16>,
@@ -599,7 +599,7 @@ struct Mma<
}
};
/// Matrix multiply-add operation: S32 = S8 * U8 + S32
/// Matrix multiply-add operation: S32 = U8 * U8 + S32
template <>
struct Mma<
gemm::GemmShape<8,8,16>,
include/cutlass/arch/sp_mma_sm80.h → include/cutlass/arch/mma_sparse_sm80.h
+9 -1
View File
@@ -29,7 +29,15 @@
#pragma once
#include "mma_sm80.h"
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "mma.h"
#include "cutlass/layout/matrix.h"
#include "cutlass/numeric_types.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
include/cutlass/arch/wmma.h
+15 -1
View File
@@ -52,7 +52,7 @@
#endif
#endif
#endif //__clang__
#endif //!defined(__clang__)
#if defined(CUTLASS_ARCH_WMMA_ENABLED)
@@ -82,6 +82,12 @@ struct CutlassToWmmaDataType<cutlass::half_t> {
using Type = __half;
};
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11)
template<>
struct CutlassToWmmaDataType<cutlass::bfloat16_t> {
using Type = __nv_bfloat16;
};
#endif
/// Statically maps int8_t => char
template<>
@@ -158,6 +164,14 @@ template<>
struct WmmaToCutlassDataType<__half> {
using Type = cutlass::half_t;
};
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDACC_VER_MAJOR__ >= 11)
template<>
struct WmmaToCutlassDataType<__nv_bfloat16> {
using Type = cutlass::bfloat16_t;
};
#endif
////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////