3.6.0 update (#2005)

* 3.6.0 update

* doc and swap stuff

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-12-25 01:34:40 -05:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent e1cd8c7866
commit 3d261a5974
258 changed files with 10863 additions and 3883 deletions
+93
View File
@@ -47,6 +47,99 @@ namespace cutlass {
namespace arch {
////////////////////////////////////////////////////////////////////////////////////////////////////
CUTLASS_DEVICE void fence_view_async_shared();
namespace detail { // namespace detail begin
// Single threaded versions that need to be called in an elect_one region
template<typename T, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array(T ptr, int arv_cnt) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
ptr[i].init(arv_cnt);
}
}
template<typename T, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array(uint64_t *ptr, int arv_cnt) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
T::init(&ptr[i], arv_cnt);
}
}
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_pair(FullBarrier full_barriers, EmptyBarrier empty_barriers, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
full_barriers[i].init(full_barrier_arv_cnt);
empty_barriers[i].init(empty_barrier_arv_cnt);
}
}
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_pair(uint64_t *full_barriers_ptr, uint64_t *empty_barriers_ptr, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
FullBarrier::init(&full_barriers_ptr[i], full_barrier_arv_cnt);
EmptyBarrier::init(&empty_barriers_ptr[i], empty_barrier_arv_cnt);
}
}
// Aligned versions that need to be call warp wide
template<typename T, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_aligned(T ptr, int arv_cnt) {
if(cute::elect_one_sync()) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
ptr[i].init(arv_cnt);
}
}
}
template<typename T, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_aligned(uint64_t *ptr, int arv_cnt) {
if(cute::elect_one_sync()) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
T::init(&ptr[i], arv_cnt);
}
}
}
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_pair_aligned(FullBarrier full_barriers, EmptyBarrier empty_barriers, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
if(cute::elect_one_sync()) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
full_barriers[i].init(full_barrier_arv_cnt);
empty_barriers[i].init(empty_barrier_arv_cnt);
}
}
}
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
CUTLASS_DEVICE
void initialize_barrier_array_pair_aligned(uint64_t *full_barriers_ptr, uint64_t *empty_barriers_ptr, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
if(cute::elect_one_sync()) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < Stages; i++) {
FullBarrier::init(&full_barriers_ptr[i], full_barrier_arv_cnt);
EmptyBarrier::init(&empty_barriers_ptr[i], empty_barrier_arv_cnt);
}
}
}
} // namespace detail end
// Enumerates the reserved named barriers to avoid potential conflicts
// This enum class specifies the NamedBarriers reserved by CUTLASS.
enum class ReservedNamedBarriers {
+4
View File
@@ -35,6 +35,8 @@
#pragma once
#include "cutlass/platform/platform.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
// SM90
@@ -79,3 +81,5 @@
/////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////
+10 -9
View File
@@ -35,6 +35,7 @@
#pragma once
#include "cutlass/array.h"
#include "cutlass/detail/helper_macros.hpp"
#include "cutlass/layout/matrix.h"
#include "cute/arch/copy_sm75.hpp"
#include "cute/arch/util.hpp"
@@ -50,7 +51,7 @@ template <
/// .x1, .x2, or .x4
int MatrixCount
>
inline __device__ void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
CUTLASS_DEVICE void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
/////////////////////////////////////////////////////////////////////////////////////////////////
//
@@ -59,19 +60,19 @@ inline __device__ void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
/////////////////////////////////////////////////////////////////////////////////////////////////
/// CUTLASS helper to get SMEM pointer
inline __device__ unsigned cutlass_get_smem_pointer(void *ptr) {
CUTLASS_DEVICE unsigned cutlass_get_smem_pointer(void *ptr) {
return cute::cast_smem_ptr_to_uint(ptr);
}
/// CUTLASS helper to get SMEM pointer
inline __device__ unsigned cutlass_get_smem_pointer(void const *ptr) {
CUTLASS_DEVICE unsigned cutlass_get_smem_pointer(void const *ptr) {
return cutlass_get_smem_pointer(const_cast<void *>(ptr));
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::RowMajor, 1>(
CUTLASS_DEVICE void ldsm<layout::RowMajor, 1>(
Array<unsigned, 1> & D,
void const* ptr) {
@@ -95,7 +96,7 @@ inline __device__ void ldsm<layout::RowMajor, 1>(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::RowMajor, 2>(
CUTLASS_DEVICE void ldsm<layout::RowMajor, 2>(
Array<unsigned, 2> & D,
void const* ptr) {
@@ -119,7 +120,7 @@ inline __device__ void ldsm<layout::RowMajor, 2>(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::RowMajor, 4>(
CUTLASS_DEVICE void ldsm<layout::RowMajor, 4>(
Array<unsigned, 4> & D,
void const* ptr) {
@@ -147,7 +148,7 @@ inline __device__ void ldsm<layout::RowMajor, 4>(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::ColumnMajor, 1>(
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 1>(
Array<unsigned, 1> & D,
void const* ptr) {
@@ -171,7 +172,7 @@ inline __device__ void ldsm<layout::ColumnMajor, 1>(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::ColumnMajor, 2>(
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 2>(
Array<unsigned, 2> & D,
void const* ptr) {
@@ -195,7 +196,7 @@ inline __device__ void ldsm<layout::ColumnMajor, 2>(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <>
inline __device__ void ldsm<layout::ColumnMajor, 4>(
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 4>(
Array<unsigned, 4> & D,
void const* ptr) {
-4
View File
@@ -33,11 +33,7 @@
*/
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "mma.h"
#include "cutlass/layout/matrix.h"
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/arch/wmma.h"
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/cutlass.h"
#include "mma.h"
-4
View File
@@ -35,11 +35,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/cutlass.h"
#include "mma.h"
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "mma.h"
#include "cutlass/layout/matrix.h"
-4
View File
@@ -35,11 +35,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "mma.h"
#include "cutlass/layout/matrix.h"
-4
View File
@@ -35,11 +35,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "mma.h"
#include "cutlass/layout/matrix.h"
+2 -2
View File
@@ -34,8 +34,8 @@
#pragma once
#include "../array.h"
#include "../numeric_types.h"
#include "cutlass/arch/array.h"
#include "cutlass/arch/numeric_types.h"
namespace cutlass {
namespace arch {
+1 -1
View File
@@ -59,7 +59,7 @@ constexpr uint32_t synclog_cap = 1 << 26;
inline std::mutex synclog_mutex;
inline std::vector<uint32_t*> synclog_buf_list;
#if defined(__NVCC__) || (defined(__clang__) && defined(__CUDA__))
inline __device__ uint32_t* synclog_buf;
CUTLASS_DEVICE uint32_t* synclog_buf;
#endif
CUTLASS_DEVICE
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/layout/matrix.h"
////////////////////////////////////////////////////////////////////////////////
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/layout/matrix.h"
////////////////////////////////////////////////////////////////////////////////
-4
View File
@@ -34,11 +34,7 @@
#pragma once
#if defined(__CUDACC_RTC__)
#include <cuda/std/cassert>
#else
#include <assert.h>
#endif
#include "cutlass/layout/matrix.h"
////////////////////////////////////////////////////////////////////////////////