3.6.0 update (#2005)
* 3.6.0 update * doc and swap stuff --------- Co-authored-by: yuzhai <yuzhai@nvidia.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
co-authored by
yuzhai
Haicheng Wu
parent
e1cd8c7866
commit
3d261a5974
@@ -47,6 +47,99 @@ namespace cutlass {
|
||||
namespace arch {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
CUTLASS_DEVICE void fence_view_async_shared();
|
||||
|
||||
namespace detail { // namespace detail begin
|
||||
|
||||
// Single threaded versions that need to be called in an elect_one region
|
||||
template<typename T, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array(T ptr, int arv_cnt) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
ptr[i].init(arv_cnt);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array(uint64_t *ptr, int arv_cnt) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
T::init(&ptr[i], arv_cnt);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_pair(FullBarrier full_barriers, EmptyBarrier empty_barriers, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
full_barriers[i].init(full_barrier_arv_cnt);
|
||||
empty_barriers[i].init(empty_barrier_arv_cnt);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_pair(uint64_t *full_barriers_ptr, uint64_t *empty_barriers_ptr, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
FullBarrier::init(&full_barriers_ptr[i], full_barrier_arv_cnt);
|
||||
EmptyBarrier::init(&empty_barriers_ptr[i], empty_barrier_arv_cnt);
|
||||
}
|
||||
}
|
||||
|
||||
// Aligned versions that need to be call warp wide
|
||||
template<typename T, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_aligned(T ptr, int arv_cnt) {
|
||||
if(cute::elect_one_sync()) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
ptr[i].init(arv_cnt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_aligned(uint64_t *ptr, int arv_cnt) {
|
||||
if(cute::elect_one_sync()) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
T::init(&ptr[i], arv_cnt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_pair_aligned(FullBarrier full_barriers, EmptyBarrier empty_barriers, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
|
||||
if(cute::elect_one_sync()) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
full_barriers[i].init(full_barrier_arv_cnt);
|
||||
empty_barriers[i].init(empty_barrier_arv_cnt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename FullBarrier, typename EmptyBarrier, uint32_t Stages>
|
||||
CUTLASS_DEVICE
|
||||
void initialize_barrier_array_pair_aligned(uint64_t *full_barriers_ptr, uint64_t *empty_barriers_ptr, int full_barrier_arv_cnt, int empty_barrier_arv_cnt) {
|
||||
if(cute::elect_one_sync()) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < Stages; i++) {
|
||||
FullBarrier::init(&full_barriers_ptr[i], full_barrier_arv_cnt);
|
||||
EmptyBarrier::init(&empty_barriers_ptr[i], empty_barrier_arv_cnt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace detail end
|
||||
|
||||
|
||||
// Enumerates the reserved named barriers to avoid potential conflicts
|
||||
// This enum class specifies the NamedBarriers reserved by CUTLASS.
|
||||
enum class ReservedNamedBarriers {
|
||||
|
||||
@@ -35,6 +35,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/platform/platform.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// SM90
|
||||
@@ -79,3 +81,5 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/array.h"
|
||||
#include "cutlass/detail/helper_macros.hpp"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
#include "cute/arch/copy_sm75.hpp"
|
||||
#include "cute/arch/util.hpp"
|
||||
@@ -50,7 +51,7 @@ template <
|
||||
/// .x1, .x2, or .x4
|
||||
int MatrixCount
|
||||
>
|
||||
inline __device__ void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
|
||||
CUTLASS_DEVICE void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
@@ -59,19 +60,19 @@ inline __device__ void ldsm(Array<unsigned, MatrixCount> & D, void const* ptr);
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// CUTLASS helper to get SMEM pointer
|
||||
inline __device__ unsigned cutlass_get_smem_pointer(void *ptr) {
|
||||
CUTLASS_DEVICE unsigned cutlass_get_smem_pointer(void *ptr) {
|
||||
return cute::cast_smem_ptr_to_uint(ptr);
|
||||
}
|
||||
|
||||
/// CUTLASS helper to get SMEM pointer
|
||||
inline __device__ unsigned cutlass_get_smem_pointer(void const *ptr) {
|
||||
CUTLASS_DEVICE unsigned cutlass_get_smem_pointer(void const *ptr) {
|
||||
return cutlass_get_smem_pointer(const_cast<void *>(ptr));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::RowMajor, 1>(
|
||||
CUTLASS_DEVICE void ldsm<layout::RowMajor, 1>(
|
||||
Array<unsigned, 1> & D,
|
||||
void const* ptr) {
|
||||
|
||||
@@ -95,7 +96,7 @@ inline __device__ void ldsm<layout::RowMajor, 1>(
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::RowMajor, 2>(
|
||||
CUTLASS_DEVICE void ldsm<layout::RowMajor, 2>(
|
||||
Array<unsigned, 2> & D,
|
||||
void const* ptr) {
|
||||
|
||||
@@ -119,7 +120,7 @@ inline __device__ void ldsm<layout::RowMajor, 2>(
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::RowMajor, 4>(
|
||||
CUTLASS_DEVICE void ldsm<layout::RowMajor, 4>(
|
||||
Array<unsigned, 4> & D,
|
||||
void const* ptr) {
|
||||
|
||||
@@ -147,7 +148,7 @@ inline __device__ void ldsm<layout::RowMajor, 4>(
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::ColumnMajor, 1>(
|
||||
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 1>(
|
||||
Array<unsigned, 1> & D,
|
||||
void const* ptr) {
|
||||
|
||||
@@ -171,7 +172,7 @@ inline __device__ void ldsm<layout::ColumnMajor, 1>(
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::ColumnMajor, 2>(
|
||||
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 2>(
|
||||
Array<unsigned, 2> & D,
|
||||
void const* ptr) {
|
||||
|
||||
@@ -195,7 +196,7 @@ inline __device__ void ldsm<layout::ColumnMajor, 2>(
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
inline __device__ void ldsm<layout::ColumnMajor, 4>(
|
||||
CUTLASS_DEVICE void ldsm<layout::ColumnMajor, 4>(
|
||||
Array<unsigned, 4> & D,
|
||||
void const* ptr) {
|
||||
|
||||
|
||||
@@ -33,11 +33,7 @@
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "mma.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "cutlass/arch/wmma.h"
|
||||
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "mma.h"
|
||||
|
||||
@@ -35,11 +35,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "mma.h"
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "mma.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
@@ -35,11 +35,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "mma.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
@@ -35,11 +35,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
|
||||
#include "mma.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
@@ -34,8 +34,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "../array.h"
|
||||
#include "../numeric_types.h"
|
||||
#include "cutlass/arch/array.h"
|
||||
#include "cutlass/arch/numeric_types.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace arch {
|
||||
|
||||
@@ -59,7 +59,7 @@ constexpr uint32_t synclog_cap = 1 << 26;
|
||||
inline std::mutex synclog_mutex;
|
||||
inline std::vector<uint32_t*> synclog_buf_list;
|
||||
#if defined(__NVCC__) || (defined(__clang__) && defined(__CUDA__))
|
||||
inline __device__ uint32_t* synclog_buf;
|
||||
CUTLASS_DEVICE uint32_t* synclog_buf;
|
||||
#endif
|
||||
|
||||
CUTLASS_DEVICE
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -34,11 +34,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
#else
|
||||
#include <assert.h>
|
||||
#endif
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Reference in New Issue
Block a user