Update fMHA kernels (#992)
* Update fMHA kernels Upstream recent changes to fMHA that we did in xFormers. Previous version in CUTLASS: facebookresearch/xformers@b6be33a Updating to: facebookresearch/xformers@55a4798 * minor changes * make var work --------- Co-authored-by: danthe3rd <danthe3rd> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
co-authored by
danthe3rd <danthe3rd>
Haicheng Wu
parent
f679663224
commit
146d314057
@@ -66,6 +66,7 @@
|
||||
#include "debug_utils.h"
|
||||
#include "epilogue/epilogue_pipelined.h"
|
||||
#include "epilogue/epilogue_rescale_output.h"
|
||||
#include "gemm/custom_mma.h"
|
||||
#include "gemm/find_default_mma.h"
|
||||
#include "gemm/mma_from_smem.h"
|
||||
#include "gemm_kernel_utils.h"
|
||||
@@ -77,7 +78,7 @@ using namespace gemm_kernel_utils;
|
||||
|
||||
namespace {
|
||||
template <typename scalar_t, typename Arch>
|
||||
constexpr int getWarpsPerSm() {
|
||||
constexpr int getWarpsPerSmFw() {
|
||||
return (
|
||||
Arch::kMinComputeCapability >= 80 &&
|
||||
!cutlass::platform::is_same<scalar_t, float>::value
|
||||
@@ -92,6 +93,24 @@ static CUTLASS_DEVICE float atomicMaxFloat(float* addr, float value) {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// If ToBatchHookType_ is supplied other than this default (which is
|
||||
// never the case in the xformers library) then the user is
|
||||
// defining the logic which each block uses to find its data to work on,
|
||||
// with the advance_to_batch function with the following signature.
|
||||
// It should return false if there is no work to do for this block.
|
||||
// In general this will not work with saving for backward due to fixed layout
|
||||
// for logsumexp and incompatible rngs for dropout, so is likely only useful for
|
||||
// custom inference.
|
||||
struct DefaultToBatchHook {
|
||||
template <typename Params>
|
||||
CUTLASS_DEVICE static bool advance_to_batch(
|
||||
Params&,
|
||||
int64_t& /* q_start */,
|
||||
int64_t& /* k_start */) {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
template <
|
||||
// The datatype of Q/K/V
|
||||
typename scalar_t_,
|
||||
@@ -99,13 +118,15 @@ template <
|
||||
typename ArchTag,
|
||||
// If Q/K/V are correctly aligned in memory and we can run a fast kernel
|
||||
bool isAligned_,
|
||||
int kQueriesPerBlock,
|
||||
int kQueriesPerBlock_,
|
||||
int kKeysPerBlock_,
|
||||
bool kSingleValueIteration_, // = `value.shape[-1] <= kKeysPerBlock`
|
||||
// upperbound on `max(value.shape[-1], query.shape[-1])`
|
||||
int kMaxK_ = (int)cutlass::platform::numeric_limits<uint32_t>::max(),
|
||||
// This is quite slower on V100 for some reason
|
||||
// Set to false if you know at compile-time you will never need dropout
|
||||
bool kSupportsDropout_ = true,
|
||||
bool kSupportsBias_ = true>
|
||||
bool kSupportsBias_ = true,
|
||||
typename ToBatchHookType_ = DefaultToBatchHook>
|
||||
struct AttentionKernel {
|
||||
enum CustomMaskType {
|
||||
NoCustomMask = 0,
|
||||
@@ -125,11 +146,14 @@ struct AttentionKernel {
|
||||
static constexpr bool kSupportsDropout = kSupportsDropout_;
|
||||
static constexpr bool kSupportsBias = kSupportsBias_;
|
||||
static constexpr int kKeysPerBlock = kKeysPerBlock_;
|
||||
static constexpr int kQueriesPerBlock = kQueriesPerBlock_;
|
||||
static constexpr int kMaxK = kMaxK_;
|
||||
static constexpr bool kIsAligned = isAligned_;
|
||||
static constexpr bool kSingleValueIteration = kSingleValueIteration_;
|
||||
static constexpr bool kSingleValueIteration = kMaxK <= kKeysPerBlock;
|
||||
static constexpr int32_t kAlignLSE = 32; // block size of backward
|
||||
static constexpr bool kPreloadV = ArchTag::kMinComputeCapability >= 80 &&
|
||||
cutlass::sizeof_bits<scalar_t>::value == 16;
|
||||
static constexpr bool kIsHalf = cutlass::sizeof_bits<scalar_t>::value == 16;
|
||||
static constexpr bool kPreloadV =
|
||||
ArchTag::kMinComputeCapability >= 80 && kIsHalf;
|
||||
static constexpr bool kKeepOutputInRF = kSingleValueIteration;
|
||||
static constexpr bool kNeedsOutputAccumulatorBuffer = !kKeepOutputInRF &&
|
||||
!cutlass::platform::is_same<output_accum_t, output_t>::value;
|
||||
@@ -143,66 +167,67 @@ struct AttentionKernel {
|
||||
// Launch bounds
|
||||
static constexpr int kNumThreads = kWarpSize * kNumWarpsPerBlock;
|
||||
static constexpr int kMinBlocksPerSm =
|
||||
getWarpsPerSm<scalar_t, ArchTag>() / kNumWarpsPerBlock;
|
||||
getWarpsPerSmFw<scalar_t, ArchTag>() / kNumWarpsPerBlock;
|
||||
|
||||
struct Params {
|
||||
// Input tensors
|
||||
scalar_t* query_ptr; // [num_queries, num_heads, head_dim]
|
||||
scalar_t* key_ptr; // [num_keys, num_heads, head_dim]
|
||||
scalar_t* value_ptr; // [num_keys, num_heads, head_dim_value]
|
||||
scalar_t* query_ptr = nullptr; // [num_queries, num_heads, head_dim]
|
||||
scalar_t* key_ptr = nullptr; // [num_keys, num_heads, head_dim]
|
||||
scalar_t* value_ptr = nullptr; // [num_keys, num_heads, head_dim_value]
|
||||
scalar_t* attn_bias_ptr = nullptr; // [num_heads, num_queries, num_keys]
|
||||
int32_t* seqstart_q_ptr = nullptr;
|
||||
int32_t* seqstart_k_ptr = nullptr;
|
||||
|
||||
int32_t* causal_diagonal_ptr = nullptr;
|
||||
int32_t* seqlen_k_ptr = nullptr;
|
||||
uint32_t causal_diagonal_offset = 0;
|
||||
|
||||
// Output tensors
|
||||
output_t* output_ptr; // [num_queries, num_heads, head_dim_value]
|
||||
output_accum_t*
|
||||
output_accum_ptr; // [num_queries, num_heads, head_dim_value]
|
||||
lse_scalar_t* logsumexp_ptr; // [num_heads, num_queries] - can be null
|
||||
output_t* output_ptr = nullptr; // [num_queries, num_heads, head_dim_value]
|
||||
// [num_queries, num_heads, head_dim_value]
|
||||
output_accum_t* output_accum_ptr = nullptr;
|
||||
// [num_heads, num_queries] - can be null
|
||||
lse_scalar_t* logsumexp_ptr = nullptr;
|
||||
|
||||
// Scale
|
||||
accum_t scale;
|
||||
accum_t scale = 0.0;
|
||||
|
||||
// Dimensions/strides
|
||||
int32_t head_dim;
|
||||
int32_t head_dim_value;
|
||||
int32_t num_queries;
|
||||
int32_t num_keys;
|
||||
int32_t head_dim = 0;
|
||||
int32_t head_dim_value = 0;
|
||||
int32_t num_queries = 0;
|
||||
int32_t num_keys = 0;
|
||||
int32_t num_keys_absolute = 0;
|
||||
|
||||
uint8_t custom_mask_type = NoCustomMask;
|
||||
|
||||
int32_t q_strideM;
|
||||
int32_t k_strideM;
|
||||
int32_t v_strideM;
|
||||
int32_t q_strideM = 0;
|
||||
int32_t k_strideM = 0;
|
||||
int32_t v_strideM = 0;
|
||||
int32_t bias_strideM = 0;
|
||||
|
||||
int32_t o_strideM = 0;
|
||||
|
||||
// Everything below is only used in `advance_to_block`
|
||||
// and shouldn't use registers
|
||||
int32_t q_strideH;
|
||||
int32_t k_strideH;
|
||||
int32_t v_strideH;
|
||||
int32_t bias_strideH = 0;
|
||||
int32_t q_strideH = 0;
|
||||
int32_t k_strideH = 0;
|
||||
int32_t v_strideH = 0;
|
||||
int64_t bias_strideH = 0;
|
||||
|
||||
int64_t q_strideB;
|
||||
int64_t k_strideB;
|
||||
int64_t v_strideB;
|
||||
int32_t bias_strideB = 0;
|
||||
int64_t q_strideB = 0;
|
||||
int64_t k_strideB = 0;
|
||||
int64_t v_strideB = 0;
|
||||
int64_t bias_strideB = 0;
|
||||
|
||||
int32_t num_batches;
|
||||
int32_t num_heads;
|
||||
int32_t num_batches = 0;
|
||||
int32_t num_heads = 0;
|
||||
|
||||
// dropout
|
||||
bool use_dropout;
|
||||
unsigned long long dropout_batch_head_rng_offset;
|
||||
float dropout_prob;
|
||||
bool use_dropout = false;
|
||||
unsigned long long dropout_batch_head_rng_offset = 0;
|
||||
float dropout_prob = 0.0f;
|
||||
#ifdef HAS_PYTORCH
|
||||
at::PhiloxCudaState rng_engine_inputs;
|
||||
at::PhiloxCudaState rng_engine_inputs = at::PhiloxCudaState(0, 0);
|
||||
#endif
|
||||
|
||||
// Moves pointers to what we should process
|
||||
@@ -220,9 +245,17 @@ struct AttentionKernel {
|
||||
head_id * num_queries * num_keys;
|
||||
}
|
||||
|
||||
int64_t q_start, k_start;
|
||||
int64_t q_start = 0, k_start = 0;
|
||||
// Advance to current batch - in case of different sequence lengths
|
||||
if (seqstart_q_ptr != nullptr) {
|
||||
constexpr bool kToBatchHook =
|
||||
!cutlass::platform::is_same<ToBatchHookType_, DefaultToBatchHook>::
|
||||
value;
|
||||
if (kToBatchHook) {
|
||||
// Call out to a custom implementation.
|
||||
if (!ToBatchHookType_::advance_to_batch(*this, q_start, k_start)) {
|
||||
return false;
|
||||
}
|
||||
} else if (seqstart_q_ptr != nullptr) {
|
||||
assert(seqstart_k_ptr != nullptr);
|
||||
seqstart_q_ptr += batch_id;
|
||||
|
||||
@@ -285,12 +318,12 @@ struct AttentionKernel {
|
||||
}
|
||||
|
||||
// Custom masking
|
||||
if (causal_diagonal_ptr) {
|
||||
causal_diagonal_offset = causal_diagonal_ptr[batch_id];
|
||||
}
|
||||
if (custom_mask_type == CausalFromBottomRight) {
|
||||
causal_diagonal_offset += num_keys - num_queries;
|
||||
causal_diagonal_offset = num_keys - num_queries;
|
||||
}
|
||||
// We use num_keys_absolute to index into the rng_state
|
||||
// We need this index to match between forward and backwards
|
||||
num_keys_absolute = num_keys;
|
||||
if (custom_mask_type == CausalFromTopLeft ||
|
||||
custom_mask_type == CausalFromBottomRight) {
|
||||
// the bottom row of the current block is query_start + kQueriesPerBlock
|
||||
@@ -323,6 +356,7 @@ struct AttentionKernel {
|
||||
|
||||
// Make sure the compiler knows these variables are the same on all
|
||||
// the threads of the warp.
|
||||
// Only worth doing if they could have been modified above.
|
||||
query_ptr = warp_uniform(query_ptr);
|
||||
key_ptr = warp_uniform(key_ptr);
|
||||
value_ptr = warp_uniform(value_ptr);
|
||||
@@ -335,8 +369,6 @@ struct AttentionKernel {
|
||||
num_queries = warp_uniform(num_queries);
|
||||
num_keys = warp_uniform(num_keys);
|
||||
num_heads = warp_uniform(num_heads);
|
||||
head_dim = warp_uniform(head_dim);
|
||||
head_dim_value = warp_uniform(head_dim_value);
|
||||
o_strideM = warp_uniform(o_strideM);
|
||||
custom_mask_type = warp_uniform(custom_mask_type);
|
||||
return true;
|
||||
@@ -395,14 +427,19 @@ struct AttentionKernel {
|
||||
ThreadblockShape, // ThreadblockShape
|
||||
WarpShape, // WarpShape
|
||||
typename GemmType::InstructionShape, // InstructionShape
|
||||
DefaultConfig::kStages, // Should use `DefaultConfig::kStages`, but that
|
||||
// uses too much smem
|
||||
ArchTag::kMinComputeCapability >= 80 && kIsHalf
|
||||
? 4
|
||||
: DefaultConfig::kStages,
|
||||
typename GemmType::Operator // Operator
|
||||
>::DefaultMma;
|
||||
using MmaCore = typename DefaultMma::MmaCore;
|
||||
using IteratorA = typename DefaultMma::IteratorA;
|
||||
using IteratorB = typename DefaultMma::IteratorB;
|
||||
using Mma = typename DefaultMma::ThreadblockMma;
|
||||
using DefaultThreadblockMma = typename DefaultMma::ThreadblockMma;
|
||||
using Mma = typename cutlass::platform::conditional<
|
||||
kSingleValueIteration,
|
||||
typename MakeCustomMma<DefaultThreadblockMma, kMaxK>::Mma,
|
||||
DefaultThreadblockMma>::type;
|
||||
using AccumLambdaIterator = typename DefaultMmaAccumLambdaIterator<
|
||||
typename Mma::Operator::IteratorC,
|
||||
accum_t,
|
||||
@@ -475,14 +512,23 @@ struct AttentionKernel {
|
||||
typename GemmType::InstructionShape,
|
||||
typename DefaultConfig::EpilogueOutputOp,
|
||||
void, // ThreadblockSwizzle - not used
|
||||
DefaultConfig::kStages,
|
||||
ArchTag::kMinComputeCapability >= 80 && kIsHalf
|
||||
? 4
|
||||
: DefaultConfig::kStages,
|
||||
false, // SplitKSerial
|
||||
typename GemmType::Operator>;
|
||||
|
||||
using WarpIteratorA = typename cutlass::gemm::threadblock::
|
||||
DefaultWarpIteratorAFromSharedMemory<
|
||||
typename DefaultGemm::Mma::Policy::Operator::Shape, // WarpShape
|
||||
typename DefaultGemm::Mma::Policy::Operator::InstructionShape,
|
||||
typename DefaultGemm::Mma::Policy::Operator::IteratorA,
|
||||
typename DefaultGemm::Mma::Policy>::WarpIterator;
|
||||
using DefaultMmaFromSmem =
|
||||
typename cutlass::gemm::threadblock::DefaultMmaFromSharedMemory<
|
||||
typename DefaultGemm::Mma,
|
||||
typename MM0::AccumulatorSharedStorage,
|
||||
MM0::AccumulatorSharedStorage::Shape::kN, // kMaxK
|
||||
WarpIteratorA,
|
||||
false>; // kScaleOperandA
|
||||
using Mma = typename DefaultMmaFromSmem::Mma;
|
||||
using IteratorB = typename Mma::IteratorB;
|
||||
@@ -500,10 +546,6 @@ struct AttentionKernel {
|
||||
typename cutlass::epilogue::threadblock::PredicatedTileIterator<
|
||||
typename DefaultEpilogue::OutputTileIterator::ThreadMap,
|
||||
output_accum_t>;
|
||||
|
||||
struct SharedStorageMM1 {
|
||||
typename Mma::SharedStorage mm;
|
||||
};
|
||||
};
|
||||
|
||||
static constexpr int64_t kAlignmentQ = MM0::kAlignmentA;
|
||||
@@ -515,6 +557,9 @@ struct AttentionKernel {
|
||||
cutlass::Array<accum_t, kQueriesPerBlock> m_prime;
|
||||
cutlass::Array<accum_t, kQueriesPerBlock> s_prime;
|
||||
cutlass::Array<accum_t, kQueriesPerBlock> mi;
|
||||
cutlass::Array<accum_t, kQueriesPerBlock> out_rescale;
|
||||
cutlass::Array<accum_t, kQueriesPerBlock * MM0::MmaCore::WarpCount::kN>
|
||||
addition_storage;
|
||||
};
|
||||
|
||||
struct SharedStorageEpilogueAtEnd : ScalingCoefs {
|
||||
@@ -524,7 +569,7 @@ struct AttentionKernel {
|
||||
typename MM0::BiasLoader::SmemTile bias;
|
||||
typename MM0::AccumulatorSharedStorage si;
|
||||
};
|
||||
typename MM1::SharedStorageMM1 mm1;
|
||||
typename MM1::Mma::SharedStorage mm1;
|
||||
};
|
||||
|
||||
union {
|
||||
@@ -546,7 +591,7 @@ struct AttentionKernel {
|
||||
typename MM0::BiasLoader::SmemTile bias;
|
||||
typename MM0::AccumulatorSharedStorage si;
|
||||
};
|
||||
typename MM1::SharedStorageMM1 mm1;
|
||||
typename MM1::Mma::SharedStorage mm1;
|
||||
typename MM1::DefaultEpilogue::SharedStorage epilogue;
|
||||
};
|
||||
|
||||
@@ -600,9 +645,6 @@ struct AttentionKernel {
|
||||
XFORMERS_CHECK(
|
||||
p.num_heads <= 1 || p.v_strideH % kAlignmentV == 0,
|
||||
"value is not correctly aligned (strideH)");
|
||||
XFORMERS_CHECK(
|
||||
p.causal_diagonal_ptr == nullptr || p.custom_mask_type != NoCustomMask,
|
||||
"`causal_diagonal_ptr` is only useful when `custom_mask_type` is causal");
|
||||
XFORMERS_CHECK(
|
||||
p.custom_mask_type < NumCustomMaskTypes,
|
||||
"invalid value for `custom_mask_type`");
|
||||
@@ -619,11 +661,13 @@ struct AttentionKernel {
|
||||
auto& m_prime = shared_storage.m_prime;
|
||||
auto& s_prime = shared_storage.s_prime;
|
||||
auto& mi = shared_storage.mi;
|
||||
auto& out_rescale = shared_storage.out_rescale;
|
||||
const uint32_t query_start = blockIdx.x * kQueriesPerBlock;
|
||||
|
||||
static_assert(kQueriesPerBlock < kNumWarpsPerBlock * kWarpSize, "");
|
||||
if (thread_id() < kQueriesPerBlock) {
|
||||
s_prime[thread_id()] = accum_t(0);
|
||||
out_rescale[thread_id()] = accum_t(1.0);
|
||||
m_prime[thread_id()] =
|
||||
-cutlass::platform::numeric_limits<accum_t>::infinity();
|
||||
mi[thread_id()] = -cutlass::platform::numeric_limits<accum_t>::infinity();
|
||||
@@ -695,7 +739,7 @@ struct AttentionKernel {
|
||||
thread_id(),
|
||||
cutlass::MatrixCoord{0, blockN * MM1::Mma::Shape::kN});
|
||||
MM1::Mma::prologue(
|
||||
shared_storage.after_mm0.mm1.mm,
|
||||
shared_storage.after_mm0.mm1,
|
||||
iterator_V,
|
||||
thread_id(),
|
||||
problem_size_1_k);
|
||||
@@ -739,7 +783,7 @@ struct AttentionKernel {
|
||||
thread_id(),
|
||||
tb_offset_B);
|
||||
|
||||
auto my_warp_id = warp_id();
|
||||
auto my_warp_id = warp_uniform(warp_id());
|
||||
auto my_lane_id = lane_id();
|
||||
|
||||
// Construct thread-scoped matrix multiply
|
||||
@@ -759,6 +803,8 @@ struct AttentionKernel {
|
||||
|
||||
if (kPreloadV) {
|
||||
prologueV(0);
|
||||
} else {
|
||||
MM1::Mma::drain_cp_asyncs();
|
||||
}
|
||||
|
||||
typename MM0::Mma::Operator::IteratorC::TensorCoord
|
||||
@@ -793,7 +839,7 @@ struct AttentionKernel {
|
||||
|
||||
// Pij += Bij, Pij is in register fragment and Bij is in shared memory
|
||||
auto lane_offset = MM0::AccumLambdaIterator::get_lane_offset(
|
||||
lane_id(), warp_id(), iteratorC_tile_offset);
|
||||
my_lane_id, my_warp_id, iteratorC_tile_offset);
|
||||
MM0::AccumLambdaIterator::iterateRows(
|
||||
lane_offset,
|
||||
[&](int accum_m) {},
|
||||
@@ -817,7 +863,7 @@ struct AttentionKernel {
|
||||
(query_start + p.causal_diagonal_offset)) {
|
||||
auto query_start = blockIdx.x * kQueriesPerBlock;
|
||||
auto lane_offset = MM0::AccumLambdaIterator::get_lane_offset(
|
||||
lane_id(), warp_id(), iteratorC_tile_offset);
|
||||
my_lane_id, my_warp_id, iteratorC_tile_offset);
|
||||
int32_t last_col;
|
||||
MM0::AccumLambdaIterator::iterateRows(
|
||||
lane_offset,
|
||||
@@ -836,30 +882,23 @@ struct AttentionKernel {
|
||||
},
|
||||
[&](int accum_m) {});
|
||||
}
|
||||
DISPATCH_BOOL(iter_key_start == 0, kIsFirst, ([&] {
|
||||
DISPATCH_BOOL(
|
||||
p.num_keys - iter_key_start >= kKeysPerBlock,
|
||||
kFullColumns,
|
||||
([&] {
|
||||
// Update `mi` from accum stored in registers
|
||||
// Also does accum[i] <- exp(accum[i] - mi)
|
||||
iterative_softmax<
|
||||
typename MM0::Mma::Operator::IteratorC,
|
||||
kFullColumns,
|
||||
kIsFirst>(
|
||||
accum_o,
|
||||
accum,
|
||||
mi,
|
||||
m_prime,
|
||||
s_prime,
|
||||
lane_id(),
|
||||
thread_id(),
|
||||
warp_id(),
|
||||
p.num_keys - iter_key_start,
|
||||
iteratorC_tile_offset,
|
||||
kSupportsBias ? 1.0f : p.scale);
|
||||
}));
|
||||
}));
|
||||
// Update `mi` from accum stored in registers
|
||||
// Also does accum[i] <- exp(accum[i] - mi)
|
||||
iterative_softmax<typename MM0::Mma::Operator::IteratorC>(
|
||||
accum_o,
|
||||
accum,
|
||||
mi,
|
||||
m_prime,
|
||||
s_prime,
|
||||
out_rescale,
|
||||
shared_storage.addition_storage,
|
||||
my_lane_id,
|
||||
thread_id(),
|
||||
my_warp_id,
|
||||
p.num_keys - iter_key_start,
|
||||
iter_key_start == 0,
|
||||
iteratorC_tile_offset,
|
||||
kSupportsBias ? 1.0f : p.scale);
|
||||
|
||||
// Output results to shared-memory
|
||||
int warp_idx_mn_0 = my_warp_id %
|
||||
@@ -910,7 +949,7 @@ struct AttentionKernel {
|
||||
curandStatePhilox4_32_10_t curand_state = curand_state_init;
|
||||
skipahead(
|
||||
static_cast<unsigned long long>(
|
||||
(query_start + thread_i) * p.num_keys +
|
||||
(query_start + thread_i) * p.num_keys_absolute +
|
||||
(iter_key_start + thread_start_j)),
|
||||
&curand_state);
|
||||
const float dropout_scale = 1.0 / (1.0 - p.dropout_prob);
|
||||
@@ -964,12 +1003,14 @@ struct AttentionKernel {
|
||||
thread_id(),
|
||||
cutlass::MatrixCoord{0, blockN * MM1::Mma::Shape::kN});
|
||||
typename MM1::Mma mma_pv(
|
||||
shared_storage.after_mm0.mm1.mm,
|
||||
shared_storage.after_mm0.si,
|
||||
// operand A: Pij_dropped in shared memory
|
||||
shared_storage.after_mm0.si.accum_ref(),
|
||||
// operand B: shared memory staging area for Vj, which is loaded
|
||||
// from global memory
|
||||
shared_storage.after_mm0.mm1.operand_B_ref(),
|
||||
(int)thread_id(),
|
||||
(int)warp_id(),
|
||||
(int)lane_id(),
|
||||
(int)problem_size_1_k);
|
||||
(int)my_warp_id,
|
||||
(int)my_lane_id);
|
||||
mma_pv.set_prologue_done(kPreloadV);
|
||||
if (!kKeepOutputInRF) {
|
||||
accum_o.clear();
|
||||
@@ -982,6 +1023,7 @@ struct AttentionKernel {
|
||||
}
|
||||
|
||||
if (!kKeepOutputInRF) {
|
||||
MM1::Mma::drain_cp_asyncs();
|
||||
DISPATCH_BOOL(
|
||||
iter_key_start == 0, kIsFirst, ([&] {
|
||||
DISPATCH_BOOL(
|
||||
@@ -1033,12 +1075,12 @@ struct AttentionKernel {
|
||||
decltype(createOutputIter),
|
||||
decltype(createOutputAccumIter)>::
|
||||
apply(createOutputIter, createOutputAccumIter, col);
|
||||
EpilogueOutputOp rescale(s_prime, m_prime);
|
||||
EpilogueOutputOp rescale(s_prime, out_rescale);
|
||||
Epilogue epilogue(
|
||||
shared_storage.epilogue_shared_storage(),
|
||||
thread_id(),
|
||||
warp_id(),
|
||||
lane_id());
|
||||
my_warp_id,
|
||||
my_lane_id);
|
||||
epilogue(rescale, dest_iter, accum_o, source_iter);
|
||||
}));
|
||||
}));
|
||||
@@ -1082,12 +1124,13 @@ struct AttentionKernel {
|
||||
typename MM1::OutputTileIteratorAccum // source tile
|
||||
>;
|
||||
auto dest_iter = createOutputIter(0);
|
||||
EpilogueOutputOp rescale(s_prime, m_prime);
|
||||
EpilogueOutputOp rescale(s_prime, out_rescale);
|
||||
Epilogue epilogue(
|
||||
shared_storage.epilogue_shared_storage(),
|
||||
thread_id(),
|
||||
warp_id(),
|
||||
lane_id());
|
||||
MM1::Mma::drain_cp_asyncs();
|
||||
epilogue(rescale, dest_iter, accum_o);
|
||||
}
|
||||
|
||||
@@ -1097,8 +1140,9 @@ struct AttentionKernel {
|
||||
static_assert(kQueriesPerBlock < kNumWarpsPerBlock * kWarpSize, "");
|
||||
if (p.logsumexp_ptr && thread_id() < kQueriesPerBlock) {
|
||||
auto lse_dim = ceil_div((int32_t)p.num_queries, kAlignLSE) * kAlignLSE;
|
||||
constexpr float kLog2e = 1.4426950408889634074; // log_2(e) = M_LOG2E
|
||||
if (thread_id() < p.num_queries) {
|
||||
p.logsumexp_ptr[thread_id()] = accum_t(mi[thread_id()]) +
|
||||
p.logsumexp_ptr[thread_id()] = accum_t(mi[thread_id()] / kLog2e) +
|
||||
cutlass::fast_log(accum_t(s_prime[thread_id()]));
|
||||
} else if (thread_id() < lse_dim) {
|
||||
p.logsumexp_ptr[thread_id()] =
|
||||
@@ -1107,20 +1151,21 @@ struct AttentionKernel {
|
||||
}
|
||||
}
|
||||
|
||||
template <
|
||||
typename WarpIteratorC,
|
||||
bool kFullColumns,
|
||||
bool kIsFirst>
|
||||
template <typename WarpIteratorC>
|
||||
CUTLASS_DEVICE static void iterative_softmax(
|
||||
typename WarpIteratorC::Fragment& frag_o, // output so far
|
||||
typename WarpIteratorC::Fragment& frag,
|
||||
cutlass::Array<accum_t, kQueriesPerBlock>& mi,
|
||||
cutlass::Array<accum_t, kQueriesPerBlock>& m_prime,
|
||||
cutlass::Array<accum_t, kQueriesPerBlock>& s_prime,
|
||||
cutlass::Array<accum_t, kQueriesPerBlock>& out_rescale,
|
||||
cutlass::Array<accum_t, kQueriesPerBlock * MM0::MmaCore::WarpCount::kN>&
|
||||
addition_storage,
|
||||
int8_t lane_id,
|
||||
int8_t thread_id,
|
||||
int8_t warp_id,
|
||||
int16_t max_col,
|
||||
int max_col,
|
||||
bool is_first,
|
||||
typename WarpIteratorC::TensorCoord const& tile_offset,
|
||||
float scaling) {
|
||||
/* Iterates on the accumulator and corresponding position on result matrix
|
||||
@@ -1141,12 +1186,11 @@ struct AttentionKernel {
|
||||
kWarpSize>::Iterator;
|
||||
// Convert to `accum_t` (rather than double)
|
||||
constexpr float kLog2e = 1.4426950408889634074; // log_2(e) = M_LOG2E
|
||||
if (!kIsFirst) {
|
||||
if (thread_id < kQueriesPerBlock) {
|
||||
m_prime[thread_id] = mi[thread_id];
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
static_assert(kQueriesPerBlock % kNumWarpsPerBlock == 0, "");
|
||||
static constexpr int kLinesPerWarp = kQueriesPerBlock / kNumWarpsPerBlock;
|
||||
|
||||
frag = cutlass::multiplies<Fragment>()(scaling * kLog2e, frag);
|
||||
|
||||
auto lane_offset =
|
||||
LambdaIterator::get_lane_offset(lane_id, warp_id, tile_offset);
|
||||
@@ -1160,46 +1204,64 @@ struct AttentionKernel {
|
||||
max = -cutlass::platform::numeric_limits<accum_t>::infinity();
|
||||
},
|
||||
[&](int accum_m, int accum_n, int idx) {
|
||||
if (kFullColumns || accum_n < max_col) {
|
||||
if (accum_n < max_col) {
|
||||
max = cutlass::fast_max(max, frag[idx]);
|
||||
}
|
||||
},
|
||||
[&](int accum_m) {
|
||||
// Having 4x atomicMax seems faster than reduce within warp
|
||||
// first...
|
||||
atomicMaxFloat(&mi[accum_m], max * scaling);
|
||||
atomicMaxFloat(&mi[accum_m], max);
|
||||
});
|
||||
}
|
||||
frag = cutlass::multiplies<Fragment>()(scaling * kLog2e, frag);
|
||||
|
||||
// Make sure we all share the update values for `mi`
|
||||
__syncthreads();
|
||||
|
||||
if (thread_id < kQueriesPerBlock) {
|
||||
auto m_prime_exp = exp2f(kLog2e * (m_prime[thread_id] - mi[thread_id]));
|
||||
m_prime[thread_id] = m_prime_exp;
|
||||
s_prime[thread_id] *= m_prime_exp;
|
||||
// Doing this `exp` is quite expensive. Let's
|
||||
// split it across the warps
|
||||
bool restore_mi_to_minus_inf = false;
|
||||
if (lane_id < kLinesPerWarp) {
|
||||
int id = warp_id * kLinesPerWarp + lane_id;
|
||||
auto m_prime_id = m_prime[id];
|
||||
auto mi_id = mi[id];
|
||||
bool changed = m_prime_id < mi_id; // `false` if both are -inf
|
||||
if (changed) {
|
||||
auto m_prime_exp = exp2f(m_prime_id - mi_id);
|
||||
out_rescale[id] = m_prime_exp;
|
||||
s_prime[id] *= m_prime_exp;
|
||||
} else {
|
||||
// Only when bias is enabled, it's possible that all the first values
|
||||
// of attention are masked to `-inf`. In that case we want to avoid
|
||||
// `nan = exp2f(-inf - (-inf))` so we temporarily set `mi` to 0
|
||||
if (kSupportsBias &&
|
||||
mi_id == -cutlass::platform::numeric_limits<accum_t>::infinity()) {
|
||||
restore_mi_to_minus_inf = true;
|
||||
mi[id] = 0.0f;
|
||||
}
|
||||
out_rescale[id] = 1.0f;
|
||||
}
|
||||
}
|
||||
__syncthreads(); // Update output fragments
|
||||
if (kKeepOutputInRF && !kIsFirst) {
|
||||
accum_t mp;
|
||||
if (kKeepOutputInRF && !is_first) {
|
||||
accum_t line_rescale;
|
||||
LambdaIterator::iterateRows(
|
||||
lane_offset,
|
||||
[&](int accum_m) { mp = m_prime[accum_m]; },
|
||||
[&](int accum_m, int accum_n, int idx) { frag_o[idx] *= mp; },
|
||||
[&](int accum_m) { line_rescale = out_rescale[accum_m]; },
|
||||
[&](int accum_m, int accum_n, int idx) {
|
||||
frag_o[idx] = frag_o[idx] * line_rescale;
|
||||
},
|
||||
[&](int accum_m) {});
|
||||
__syncthreads();
|
||||
}
|
||||
// Update accum_m, accum_n, ...
|
||||
{
|
||||
accum_t mi_row, total_row;
|
||||
LambdaIterator::iterateRows(
|
||||
lane_offset,
|
||||
[&](int accum_m) { mi_row = kLog2e * mi[accum_m]; },
|
||||
[&](int accum_m) { mi_row = mi[accum_m]; },
|
||||
[&](int accum_m, int accum_n, int idx) {
|
||||
frag[idx] = (kFullColumns || accum_n < max_col)
|
||||
? exp2f(frag[idx] - mi_row)
|
||||
: accum_t(0.0);
|
||||
frag[idx] =
|
||||
(accum_n < max_col) ? exp2f(frag[idx] - mi_row) : accum_t(0.0);
|
||||
},
|
||||
[&](int accum_m) {});
|
||||
LambdaIterator::iterateRows(
|
||||
@@ -1211,10 +1273,30 @@ struct AttentionKernel {
|
||||
lane_id, total_row, [](accum_t a, accum_t b) {
|
||||
return a + b;
|
||||
})) {
|
||||
atomicAdd(&s_prime[accum_m], total_row);
|
||||
// NOTE: we could atomically add `total_row` to `s_prime`, but
|
||||
// it's faster (and deterministic) to avoid atomics here
|
||||
addition_storage
|
||||
[accum_m + kQueriesPerBlock * tile_offset.column()] =
|
||||
total_row;
|
||||
}
|
||||
});
|
||||
}
|
||||
__syncthreads();
|
||||
if (lane_id < kLinesPerWarp) {
|
||||
int id = warp_id * kLinesPerWarp + lane_id;
|
||||
accum_t total_row = s_prime[id];
|
||||
if (restore_mi_to_minus_inf) {
|
||||
// Restore `mi`, see above when we set `restore_mi_to_minus_inf=true`
|
||||
mi[id] = -cutlass::platform::numeric_limits<accum_t>::infinity();
|
||||
} else {
|
||||
m_prime[id] = mi[id];
|
||||
}
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < MM0::MmaCore::WarpCount::kN; ++i) {
|
||||
total_row += addition_storage[id + kQueriesPerBlock * i];
|
||||
}
|
||||
s_prime[id] = total_row;
|
||||
}
|
||||
}
|
||||
|
||||
static CUTLASS_DEVICE int8_t lane_id() {
|
||||
|
||||
Reference in New Issue
Block a user