CUTLASS 3.8 Release (#2059)
* CUTLASS 3.8 Release * update * Update README.md * Revert "Update README.md" This reverts commit b353e36fe83e0815f99b44e46c0c95494c44726b. * update * update --------- Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
mihir-awatramani
co-authored by
Haicheng Wu
Haicheng Wu
Haicheng Wu
Haicheng Wu
parent
9eb01fa0b0
commit
389e493055
@@ -41,6 +41,7 @@
|
||||
namespace cutlass {
|
||||
namespace arch {
|
||||
|
||||
constexpr int sm100_smem_capacity_bytes = 232448;
|
||||
#if defined(__NVCC__) || defined(__CUDACC_RTC__) || (defined(__clang__) && defined(__CUDA__))
|
||||
|
||||
/// Computes laneId within a warp
|
||||
@@ -93,6 +94,12 @@ struct Sm90 {
|
||||
static int const kMinComputeCapability = 90;
|
||||
};
|
||||
|
||||
|
||||
struct Sm100 {
|
||||
static int const kMinComputeCapability = 100;
|
||||
};
|
||||
|
||||
|
||||
/// Triggers a breakpoint on the device
|
||||
CUTLASS_DEVICE
|
||||
void device_breakpoint() {
|
||||
|
||||
@@ -36,12 +36,21 @@
|
||||
|
||||
#include <cutlass/arch/memory_sm75.h>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
#include <cute/arch/copy_sm100_tma.hpp>
|
||||
#include <cutlass/arch/config.h>
|
||||
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 && (__CUDACC_VER_MAJOR__ >= 12)
|
||||
#define CUDA_BARRIER_ENABLED 1
|
||||
#else
|
||||
#define CUDA_BARRIER_ENABLED 0
|
||||
#endif
|
||||
|
||||
|
||||
#if (defined(CUTLASS_ARCH_MMA_SM100A_ENABLED))
|
||||
#define CUTLASS_ARCH_TCGEN_ENABLED 1
|
||||
#endif
|
||||
|
||||
|
||||
namespace cutlass {
|
||||
/// @brief
|
||||
namespace arch {
|
||||
@@ -140,6 +149,15 @@ void initialize_barrier_array_pair_aligned(uint64_t *full_barriers_ptr, uint64_t
|
||||
} // namespace detail end
|
||||
|
||||
|
||||
|
||||
|
||||
// There are 16 Named Barriers provided by Hardware starting in Hopper
|
||||
// Their IDs are in the range 0-15
|
||||
// Number of threads syncing using the barrier must be a multiple of warp-size
|
||||
// ID 0 should not be used for safety, as other driver APIs (i.e. __syncthreads)
|
||||
// may use it and conflict with other uses.
|
||||
|
||||
|
||||
// Enumerates the reserved named barriers to avoid potential conflicts
|
||||
// This enum class specifies the NamedBarriers reserved by CUTLASS.
|
||||
enum class ReservedNamedBarriers {
|
||||
@@ -148,6 +166,7 @@ enum class ReservedNamedBarriers {
|
||||
TransformBarrier = 3,
|
||||
StreamkBarrier0 = 4,
|
||||
StreamkBarrier1 = 5
|
||||
, TmemAllocBarrier = 6
|
||||
, FirstUserBarrier = StreamkBarrier1 + 1
|
||||
};
|
||||
|
||||
@@ -735,6 +754,152 @@ void cpasync_barrier_arrive_noinc(uint64_t const* smem_ptr) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive(uint64_t const* smem_ptr) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
if (cute::elect_one_sync()) {
|
||||
asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];"
|
||||
:
|
||||
:"r"(bar_intptr));
|
||||
}
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
//UMMA arrive for MMA_2x1SM
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_2x1SM(uint64_t const* smem_ptr) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
if (cute::elect_one_sync()) {
|
||||
asm volatile("tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];"
|
||||
:
|
||||
:"r"(bar_intptr));
|
||||
}
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
// UMMA arrive for MMA_1sm + TMA_LOAD_MULTICAST combination
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_multicast(uint64_t const* smem_ptr, uint16_t cta_mask) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
if(cute::elect_one_sync()) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1; \n\t"
|
||||
"}"
|
||||
:
|
||||
:"r"(bar_intptr), "h"(cta_mask));
|
||||
}
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
// UMMA arrive for MMA_2x1SM + TMA_LOAD_MULTICAST combination
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_multicast_2x1SM(uint64_t const* smem_ptr, uint16_t cta_mask) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
if (cute::elect_one_sync()) {
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1; \n\t"
|
||||
"}"
|
||||
:
|
||||
:"r"(bar_intptr), "h"(cta_mask));
|
||||
}
|
||||
#else
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Temporary solution for sparse kernel.
|
||||
// Will remove this when we done tightly elect_one wrap.
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_multicast_no_elect(uint64_t const* smem_ptr, uint16_t cta_mask) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .b16 lo, hi;\n\t"
|
||||
"mov.b32 {lo, hi}, %1;\n\t"
|
||||
"tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], lo; \n\t"
|
||||
"}"
|
||||
:
|
||||
:"r"(bar_intptr), "r"(uint32_t(cta_mask)));
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
CUTLASS_NOT_IMPLEMENTED();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Temporary solution for sparse kernel.
|
||||
// UMMA arrive for MMA_2x1SM + TMA_LOAD_MULTICAST combination
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_multicast_2x1SM_no_elect(uint64_t const* smem_ptr, uint16_t cta_mask) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr);
|
||||
asm volatile(
|
||||
"{\n\t"
|
||||
".reg .b16 lo, hi;\n\t"
|
||||
"mov.b32 {lo, hi}, %1;\n\t"
|
||||
"tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], lo; \n\t"
|
||||
"}"
|
||||
:
|
||||
:"r"(bar_intptr), "r"(uint32_t(cta_mask)));
|
||||
#else
|
||||
CUTLASS_NOT_IMPLEMENTED();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Always arrive on even SM of collaborating 2 SMs.
|
||||
CUTLASS_DEVICE
|
||||
void umma_arrive_2x1SM_sm0(uint64_t const* smem_ptr) {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
uint32_t bar_intptr = cute::cast_smem_ptr_to_uint(smem_ptr) & cute::Sm100MmaPeerBitMask;
|
||||
asm volatile (
|
||||
"{\n\t"
|
||||
"mbarrier.arrive.shared::cluster.b64 _, [%0];\n\t"
|
||||
"}"
|
||||
:
|
||||
: "r"(bar_intptr));
|
||||
|
||||
#else
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
CUTE_DEVICE static void fence_view_async_tmem_load() {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
asm volatile (
|
||||
"{\n\t"
|
||||
"tcgen05.wait::ld.sync.aligned; \n"
|
||||
"}"
|
||||
::);
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
CUTE_DEVICE static void fence_view_async_tmem_store() {
|
||||
#if defined(CUTLASS_ARCH_TCGEN_ENABLED)
|
||||
asm volatile (
|
||||
"{\n\t"
|
||||
"tcgen05.wait::st.sync.aligned; \n"
|
||||
"}"
|
||||
::);
|
||||
#elif defined(__CUDA_ARCH__)
|
||||
asm volatile ("brkpt;\n" ::);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // end namespace arch
|
||||
} // end namespace cutlass
|
||||
|
||||
@@ -51,21 +51,32 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 2)
|
||||
#if (__CUDACC_VER_MAJOR__ > 12 || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 2))
|
||||
#define CUTLASS_ARCH_MMA_SPARSE_SM90_SUPPORTED
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// SM90 Modifiable
|
||||
// SM90 Modifiable TMA
|
||||
#if (__CUDACC_VER_MAJOR__ > 12 || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 3))
|
||||
#define CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED 1
|
||||
#if (!defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 900)
|
||||
#if (!defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900)
|
||||
#define CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if (!defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90A_ENABLED) && defined(__CUDA_ARCH_FEAT_SM90_ALL))
|
||||
#define CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90A_ENABLED 1
|
||||
#if (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ == 8)
|
||||
#if defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED)
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 900 && \
|
||||
!defined(__CUDA_ARCH_FEAT_SM90_ALL)
|
||||
#undef CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED
|
||||
#endif
|
||||
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 1000 && \
|
||||
!defined(__CUDA_ARCH_FEAT_SM100_ALL)
|
||||
#undef CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_ENABLED
|
||||
#endif
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -79,7 +90,29 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// SM100, SM100a
|
||||
#if !CUTLASS_CLANG_CUDA && (__CUDACC_VER_MAJOR__ > 12 || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 8))
|
||||
#define CUTLASS_ARCH_MMA_SM100_SUPPORTED 1
|
||||
#if (!defined(CUTLASS_ARCH_MMA_SM100_ENABLED) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 1000)
|
||||
#define CUTLASS_ARCH_MMA_SM100_ENABLED 1
|
||||
|
||||
#if (!defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) && defined(__CUDA_ARCH_FEAT_SM100_ALL))
|
||||
#define CUTLASS_ARCH_MMA_SM100A_ENABLED 1
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
#if (defined(CUTLASS_ARCH_MMA_SM100A_ENABLED))
|
||||
# define CUTLASS_ARCH_CLC_ENABLED
|
||||
#endif
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -129,6 +129,11 @@ struct OpClassWmmaTensorOp {};
|
||||
/// Tag classifying operators as Tensor Core with structure sparse operations.
|
||||
struct OpClassSparseTensorOp {};
|
||||
|
||||
|
||||
/// Tag classifying operators as Tensor Core with blockScaled
|
||||
struct OpClassBlockScaledTensorOp {};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Matrix multiply-add operation
|
||||
|
||||
Reference in New Issue
Block a user