@@ -282,5 +282,52 @@ inline __device__ void ldsm<layout::ColumnMajor, 4>(
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename AccessType, int Bytes>
|
||||
struct shared_load_op {
|
||||
CUTLASS_DEVICE
|
||||
shared_load_op(AccessType &D, void const *ptr) {
|
||||
D = *reinterpret_cast<AccessType const *>(ptr);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename AccessType>
|
||||
CUTLASS_DEVICE void shared_load(AccessType &D, void const *ptr) {
|
||||
shared_load_op<AccessType, int(sizeof(AccessType))>(D, ptr);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename AccessType>
|
||||
struct shared_load_op<AccessType, 16> {
|
||||
CUTLASS_DEVICE
|
||||
shared_load_op(AccessType &D, void const *ptr) {
|
||||
unsigned addr = cutlass_get_smem_pointer(ptr);
|
||||
|
||||
uint4 v;
|
||||
asm volatile ("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];" :
|
||||
"=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "r"(addr));
|
||||
|
||||
D = reinterpret_cast<AccessType const &>(v);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename AccessType>
|
||||
struct shared_load_op<AccessType, 8> {
|
||||
CUTLASS_DEVICE
|
||||
shared_load_op(AccessType &D, void const *ptr) {
|
||||
unsigned addr = cutlass_get_smem_pointer(ptr);
|
||||
|
||||
uint2 v;
|
||||
asm volatile ("ld.shared.v2.b32 {%0, %1}, [%2];" :
|
||||
"=r"(v.x), "=r"(v.y) : "r"(addr));
|
||||
|
||||
D = reinterpret_cast<AccessType const &>(v);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace arch
|
||||
} // namespace cutlass
|
||||
|
||||
@@ -68,6 +68,18 @@ template <
|
||||
CacheOperation::Kind cache_op = CacheOperation::Always>
|
||||
struct cp_async_zfill;
|
||||
|
||||
/// Initiates an asynchronous copy from global memory to shared memory. Rather than predicate
|
||||
/// the entire transfer, nans (0x7eff) are written to SMEM if the guard predicate is false.
|
||||
///
|
||||
/// LDGSTS
|
||||
///
|
||||
template <
|
||||
/// Size of the access in bytes
|
||||
int SizeInBytes,
|
||||
/// Cache operation
|
||||
CacheOperation::Kind cache_op = CacheOperation::Always>
|
||||
struct cp_async_nan;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Partial specialization
|
||||
@@ -150,6 +162,48 @@ struct cp_async_zfill<SizeInBytes, CacheOperation::Always> {
|
||||
}
|
||||
};
|
||||
|
||||
__device__ __constant__ uint4 OOB_NAN_F16x8 = {0x7eff7eff, 0x7eff7eff,
|
||||
0x7eff7eff, 0x7eff7eff};
|
||||
|
||||
/// Partial specialization
|
||||
template <>
|
||||
struct cp_async_nan<16, CacheOperation::Always> {
|
||||
static int const kSizeInBytes = 16;
|
||||
|
||||
/// Copy with nan fill
|
||||
CUTLASS_DEVICE
|
||||
cp_async_nan(void *smem_ptr, void const *global_ptr, bool pred_guard) {
|
||||
#if CUDA_CP_ASYNC_ACTIVATED
|
||||
|
||||
unsigned smem_int_ptr = cutlass_get_smem_pointer(smem_ptr);
|
||||
|
||||
asm volatile(
|
||||
"{\n"
|
||||
" .reg .pred p;\n"
|
||||
" setp.ne.b32 p, %0, 0;\n"
|
||||
#if CUTLASS_ENABLE_L2_PREFETCH
|
||||
" @p cp.async.ca.shared.global.L2::128B [%1], [%2], %3;\n"
|
||||
#else
|
||||
" @p cp.async.ca.shared.global [%1], [%2], %3;\n"
|
||||
#endif
|
||||
" @!p st.shared.v4.u32 [%1], {%4, %5, %6, %7};\n"
|
||||
"}\n"
|
||||
:
|
||||
: "r"((int)pred_guard), "r"(smem_int_ptr), "l"(global_ptr),
|
||||
"n"(kSizeInBytes), "r"(OOB_NAN_F16x8.x), "r"(OOB_NAN_F16x8.y), "r"(OOB_NAN_F16x8.z),
|
||||
"r"(OOB_NAN_F16x8.w));
|
||||
|
||||
#else
|
||||
|
||||
CUTLASS_UNUSED(smem_ptr);
|
||||
CUTLASS_UNUSED(global_ptr);
|
||||
CUTLASS_UNUSED(pred_guard);
|
||||
CUTLASS_NOT_IMPLEMENTED();
|
||||
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Partial specialization
|
||||
|
||||
@@ -62,6 +62,16 @@ struct OpMultiplyAddFastF16;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Tag indicating the input is converted to 2 (big and small) TF32 components
|
||||
// Perform 3xTF32 or 4xTF32 for every F32 output element
|
||||
struct OpMultiplyAddFastF32;
|
||||
|
||||
/// Tag indicating the input is converted to 2 (big and small) TF32 components
|
||||
// Perform 3xTF32 or 4xTF32 for every complex<F32> output element
|
||||
struct OpMultiplyAddComplexFastF32;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Tag indicating the complex multiply-add operation
|
||||
struct OpMultiplyAddComplex;
|
||||
|
||||
|
||||
@@ -1170,7 +1170,6 @@ struct Mma<
|
||||
) const {
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM75_ENABLED)
|
||||
|
||||
#if defined(CUTLASS_ARCH_WMMA_ENABLED)
|
||||
using WmmaFragmentA = nvcuda::wmma::fragment<
|
||||
nvcuda::wmma::matrix_a,
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "mma.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
@@ -2109,7 +2110,6 @@ struct Mma<
|
||||
|
||||
int const *C = reinterpret_cast<int const *>(&c);
|
||||
int *D = reinterpret_cast<int *>(&d);
|
||||
|
||||
asm volatile(
|
||||
"mma.sync.aligned.m16n8k256.row.col.s32.b1.b1.s32.xor.popc {%0,%1,%2,%3}, "
|
||||
"{%4,%5,%6,%7}, "
|
||||
@@ -2119,8 +2119,10 @@ struct Mma<
|
||||
"r"(C[0]), "r"(C[1]), "r"(C[2]), "r"(C[3]));
|
||||
|
||||
#else
|
||||
|
||||
assert(0);
|
||||
#endif
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM80_ENABLED)
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user