CUTLASS 3.5.1 (#1623)

* CUTLASS 3.5.1

* updates, optimizations, fixes
This commit is contained in:
Vijay Thakkar
2024-07-29 08:46:24 -04:00
committed by GitHub
parent 56b46e2d13
commit be60a0b272
312 changed files with 19793 additions and 6775 deletions
+86 -42
View File
@@ -49,7 +49,6 @@ using namespace cute;
enum class BarrierStatus : uint32_t {
WaitAgain = 0u,
WaitDone = 1u,
WaitOnly = 2u
};
class ArrivalToken {
@@ -62,7 +61,7 @@ public:
CUTLASS_HOST_DEVICE
BarrierStatus get() const {
return barrier_status_;;
return barrier_status_;
}
CUTLASS_HOST_DEVICE
@@ -241,7 +240,7 @@ public :
, full_barrier_ptr_(&storage.full_barrier_[0])
, empty_barrier_ptr_(&storage.empty_barrier_[0]) {
int warp_idx = canonical_warp_idx();
int warp_idx = canonical_warp_idx_sync();
int lane_predicate = cute::elect_one_sync();
if (warp_idx == 0 && lane_predicate == 1) {
@@ -344,7 +343,7 @@ public :
CUTLASS_DEVICE
void producer_tail(PipelineState state) {
for (int count = 0; count < Stages; ++count) {
producer_acquire(state, {BarrierStatus::WaitOnly});
empty_barrier_ptr_[state.index()].wait(state.phase());
++state;
}
}
@@ -394,7 +393,7 @@ private :
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -403,9 +402,6 @@ private :
if (barrier_token != BarrierStatus::WaitDone) {
empty_barrier_ptr_[stage].wait(phase);
}
if (barrier_token == BarrierStatus::WaitOnly) {
return;
}
if (params_.is_leader) {
full_barrier_ptr_[stage].arrive_and_expect_tx(params_.transaction_bytes);
@@ -456,7 +452,7 @@ private :
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -465,7 +461,7 @@ private :
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].test_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].test_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -659,7 +655,7 @@ public :
, full_barrier_ptr_(storage.full_barrier_.data())
, empty_barrier_ptr_(storage.empty_barrier_.data()) {
int warp_idx = canonical_warp_idx();
int warp_idx = canonical_warp_idx_sync();
int lane_predicate = cute::elect_one_sync();
// Barrier FULL, EMPTY init
@@ -761,7 +757,7 @@ private:
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -793,7 +789,7 @@ private:
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -802,7 +798,7 @@ private:
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].test_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].test_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -824,20 +820,31 @@ private:
// Simple producer-consumer async Pipeline class
//
///////////////////////////////////////////////////////////////////////////////////////////////////
template <int Stages_>
class PipelineAsync {
public :
using FullBarrier = cutlass::arch::ClusterBarrier;
using EmptyBarrier = cutlass::arch::ClusterBarrier;
using ProducerBarrierType = FullBarrier::ValueType;
using ConsumerBarrierType = EmptyBarrier::ValueType;
static constexpr uint32_t Stages = Stages_;
using PipelineState = cutlass::PipelineState<Stages>;
struct SharedStorage {
namespace PipelineDetail {
template<int Stages>
using PipelineAsyncPipelineState = cutlass::PipelineState<Stages>;
template<int Stages>
struct PipelineAsyncSharedStorage {
using FullBarrier = cutlass::arch::ClusterBarrier;
using EmptyBarrier = cutlass::arch::ClusterBarrier;
FullBarrier full_barrier_[Stages];
EmptyBarrier empty_barrier_[Stages];
};
};
template <int Stages_>
class PipelineAsync {
public :
static constexpr uint32_t Stages = Stages_;
using SharedStorage = PipelineDetail::PipelineAsyncSharedStorage<Stages>;
using FullBarrier = typename SharedStorage::FullBarrier;
using EmptyBarrier = typename SharedStorage::EmptyBarrier;
using ProducerBarrierType = typename FullBarrier::ValueType;
using ConsumerBarrierType = typename EmptyBarrier::ValueType;
using PipelineState = PipelineDetail::PipelineAsyncPipelineState<Stages>;
enum class ThreadCategory {
NonParticipant,
@@ -867,7 +874,7 @@ public :
full_barrier_ptr_(&storage.full_barrier_[0]),
empty_barrier_ptr_(&storage.empty_barrier_[0]) {
int warp_idx = canonical_warp_idx();
int warp_idx = canonical_warp_idx_sync();
int lane_predicate = cute::elect_one_sync();
// Barrier FULL, EMPTY init
@@ -960,6 +967,11 @@ public :
consumer_release(state.index());
}
CUTLASS_DEVICE
ProducerBarrierType* producer_get_barrier(uint32_t stage) {
return reinterpret_cast<ProducerBarrierType*>(&full_barrier_ptr_[stage]);
}
private:
Params params_;
FullBarrier *full_barrier_ptr_;
@@ -970,7 +982,7 @@ private:
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = empty_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -986,17 +998,12 @@ private:
full_barrier_ptr_[stage].arrive();
}
CUTLASS_DEVICE
ProducerBarrierType* producer_get_barrier(uint32_t stage) {
return reinterpret_cast<ProducerBarrierType*>(&full_barrier_ptr_[stage]);
}
CUTLASS_DEVICE
ConsumerToken consumer_try_wait(uint32_t stage, uint32_t phase, uint32_t skip_wait) {
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].try_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].try_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
@@ -1005,13 +1012,13 @@ private:
if (skip_wait) {
return {BarrierStatus::WaitDone};
}
uint32_t barrier_status = full_barrier_ptr_[stage].test_wait(phase);
bool barrier_status = full_barrier_ptr_[stage].test_wait(phase);
return {static_cast<BarrierStatus>(barrier_status)};
}
CUTLASS_DEVICE
void consumer_wait(uint32_t stage, uint32_t phase) {
uint32_t done = full_barrier_ptr_[stage].test_wait(phase);
bool done = full_barrier_ptr_[stage].test_wait(phase);
if (!done) {
full_barrier_ptr_[stage].wait(phase);
}
@@ -1040,14 +1047,24 @@ private:
//
///////////////////////////////////////////////////////////////////////////////////////////////////
template<int SequenceDepth, int SequenceLength>
class OrderedSequenceBarrier {
public :
using Barrier = cutlass::arch::ClusterBarrier;
namespace PipelineDetail {
struct SharedStorage {
Barrier barrier_[SequenceDepth][SequenceLength];
};
template<int SequenceDepth, int SequenceLength>
struct OrderedSequenceBarrierSharedStorage {
using Barrier = cutlass::arch::ClusterBarrier;
Barrier barrier_[SequenceDepth][SequenceLength];
};
} // namespace PipelineDetail
template<int SequenceDepth_, int SequenceLength_>
class OrderedSequenceBarrier {
public:
static constexpr int SequenceDepth = SequenceDepth_;
static constexpr int SequenceLength = SequenceLength_;
using SharedStorage =
PipelineDetail::OrderedSequenceBarrierSharedStorage<SequenceDepth, SequenceLength>;
using Barrier = typename SharedStorage::Barrier;
struct Params {
uint32_t group_id;
@@ -1077,7 +1094,7 @@ public:
barrier_ptr_(&storage.barrier_[0][0]),
// Group 0 - starts with an opposite phase
stage_({0, params.group_id == 0, 0}) {
int warp_idx = canonical_warp_idx();
int warp_idx = canonical_warp_idx_sync();
int lane_predicate = cute::elect_one_sync();
// Barrier FULL, EMPTY init
@@ -1122,4 +1139,31 @@ private:
////////////////////////////////////////////////////////////////////////////////////////////////////
// Synchronization call. Blocks until barriers are initialized in shared memory.
CUTLASS_DEVICE
void
pipeline_init_wait(int cluster_size) {
if (cluster_size > 1) {
cute::cluster_wait();
}
else {
__syncthreads();
}
}
// Used to guarantee that the Pipeline init is visible
// to all producers and consumer threadblocks in the cluster
CUTLASS_DEVICE
void
pipeline_init_arrive_relaxed(int cluster_size) {
if (cluster_size > 1) {
cute::cluster_arrive_relaxed();
}
else {
__syncthreads();
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
} // end namespace cutlass