Collection of changes to fix clang build. (#1200)
* Remove unused variables * Qualify calls to make_fragment_? from templated base class. Fixes clang build error. * Add missing `#include <cstdio>` * Various changes to fix clang compile errors. * More changes to fix clang build. Remaining issues: - `params` initializer of `CollectiveEpilogue`. - `ops` initializer of `Sm90VisitorImplBase`. - `__usAtomicCAS` needs to be added to clang upstream. * Fix remaining clang build issues. * Qualify `cute::rank()` calls. * Qualify some more calls that are otherwise ambiguous between `cute` and `std` namespace. * Double-escape special registers in inline asm. * small change --------- Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
co-authored by
Haicheng Wu
parent
f4a0216601
commit
e1483d5fa0
@@ -595,7 +595,7 @@ CollectiveBuilder<
|
||||
cute::is_base_of_v<TmaWarpSpecializedCooperativeElementwiseBase, Schedule> >> {
|
||||
private:
|
||||
using FusionOp =
|
||||
fusion::LinCombEltAct<Schedule::ActivationFunctor, ElementD, ElementCompute, ElementCompute, Schedule::Round>;
|
||||
fusion::LinCombEltAct<Schedule::template ActivationFunctor, ElementD, ElementCompute, ElementCompute, Schedule::Round>;
|
||||
using ImplSchedule =
|
||||
cute::conditional_t<cute::is_base_of_v<TmaWarpSpecializedElementwiseBase, Schedule>,
|
||||
TmaWarpSpecialized, TmaWarpSpecializedCooperative>;
|
||||
@@ -676,7 +676,7 @@ private:
|
||||
using SmemCopyOpAux = decltype(detail::sm90_get_smem_store_op_for_accumulator<
|
||||
GmemStrideTypeAux, typename Schedule::ElementT>());
|
||||
using FusionOperationAux = fusion::LinCombPerRowBiasEltActAux<
|
||||
GmemLayoutTagD, Schedule::ActivationFunctor, ElementD, ElementCompute,
|
||||
GmemLayoutTagD, Schedule::template ActivationFunctor, ElementD, ElementCompute,
|
||||
typename Schedule::ElementT, typename Schedule::ElementBias, ElementCompute
|
||||
>;
|
||||
using FusionCallbacksAux = fusion::FusionCallbacks<
|
||||
@@ -684,7 +684,7 @@ private:
|
||||
>;
|
||||
|
||||
using FusionOperationNoAux = fusion::LinCombPerRowBiasEltAct<
|
||||
Schedule::ActivationFunctor, ElementD, ElementCompute,
|
||||
Schedule::template ActivationFunctor, ElementD, ElementCompute,
|
||||
typename Schedule::ElementBias, ElementCompute
|
||||
>;
|
||||
using FusionCallbacksNoAux = fusion::FusionCallbacks<
|
||||
|
||||
@@ -81,8 +81,8 @@ public:
|
||||
static const int kOutputAlignment = ThreadEpilogueOp::kCount;
|
||||
using AlignmentType = typename cute::uint_bit<sizeof_bits<ElementOutput>::value * kOutputAlignment>::type;
|
||||
|
||||
static_assert(rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
|
||||
struct SharedStorage { };
|
||||
|
||||
@@ -163,10 +163,10 @@ public:
|
||||
using namespace cute;
|
||||
using X = Underscore;
|
||||
|
||||
static_assert(rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(cute::rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(is_static<BlockShapeMNK>::value, "ThreadBlock tile shape must be static");
|
||||
static_assert(rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 3");
|
||||
static_assert(cute::rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(cute::rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 3");
|
||||
|
||||
// Separate out problem shape for convenience
|
||||
auto M = get<0>(problem_shape_mnkl);
|
||||
|
||||
@@ -204,12 +204,12 @@ public:
|
||||
int thread_idx,
|
||||
TensorStorage& shared_tensors)
|
||||
{
|
||||
constexpr int BLK_M_RANK = rank<0>(tile_shape_MNK);
|
||||
constexpr int BLK_M_RANK = cute::rank<0>(tile_shape_MNK);
|
||||
auto m_max_coord = unwrap(cute::transform(make_seq<BLK_M_RANK>{}, [&](auto i) {
|
||||
return get<0,i>(problem_shape_mnkl) - get<0,i>(tile_shape_MNK) * get<0,i>(tile_coord_mnkl);
|
||||
}));
|
||||
|
||||
constexpr int BLK_N_RANK = rank<1>(tile_shape_MNK);
|
||||
constexpr int BLK_N_RANK = cute::rank<1>(tile_shape_MNK);
|
||||
auto n_max_coord = unwrap(cute::transform(make_seq<BLK_N_RANK>{}, [&](auto i) {
|
||||
return get<1,i>(problem_shape_mnkl) - get<1,i>(tile_shape_MNK) * get<1,i>(tile_coord_mnkl);
|
||||
}));
|
||||
|
||||
@@ -91,8 +91,8 @@ public:
|
||||
using StrideD = StrideD_;
|
||||
using ActivationFunctor = typename ThreadEpilogueOp::ActivationFunctor;
|
||||
|
||||
static_assert(rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
|
||||
static constexpr int kOutputAlignment = ThreadEpilogueOp::kCount;
|
||||
using AlignmentType = typename cute::uint_bit<sizeof_bits<ElementOutput>::value * kOutputAlignment>::type;
|
||||
@@ -182,10 +182,10 @@ public:
|
||||
using namespace cute;
|
||||
using X = Underscore;
|
||||
|
||||
static_assert(rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(cute::rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(is_static<BlockShapeMNK>::value, "ThreadBlock tile shape must be static");
|
||||
static_assert(rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 4");
|
||||
static_assert(cute::rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(cute::rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 4");
|
||||
|
||||
// Separate out problem shape for convenience
|
||||
auto M = get<0>(problem_shape_mnkl);
|
||||
|
||||
@@ -87,8 +87,8 @@ public:
|
||||
static const int kOutputAlignment = ThreadEpilogueOp::kCount;
|
||||
using AlignmentType = typename cute::uint_bit<sizeof_bits<ElementOutput>::value * kOutputAlignment>::type;
|
||||
|
||||
static_assert(rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
|
||||
struct SharedStorage
|
||||
{
|
||||
@@ -172,10 +172,10 @@ public:
|
||||
using namespace cute;
|
||||
using X = Underscore;
|
||||
|
||||
static_assert(rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(cute::rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(is_static<BlockShapeMNK>::value, "ThreadBlock tile shape must be static");
|
||||
static_assert(rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 3");
|
||||
static_assert(cute::rank(BlockShapeMNK{}) == 3, "BlockShapeMNK must be rank 3");
|
||||
static_assert(cute::rank(BlockCoordMNKL{}) == 4, "BlockCoordMNKL must be rank 3");
|
||||
|
||||
// synchronizing function for smem reads/writes
|
||||
#if CUDA_BARRIER_ENABLED
|
||||
|
||||
@@ -113,12 +113,12 @@ public:
|
||||
using GmemTiledCopyD = SM90_TMA_STORE;
|
||||
|
||||
static_assert(!is_layout<EpilogueTile>::value && is_tuple<EpilogueTile>::value, "EpilogueTile must be a cute::Tile or cute::Shape");
|
||||
static_assert(rank(CtaTileMNK{}) == 3, "CtaTileMNK must be rank-3: [CTA_M, CTA_N, CTA_K]");
|
||||
static_assert(rank(EpilogueTile{}) == 2, "EpilogueTile must be rank-2: [EPI_TILE_M, EPI_TILE_N]");
|
||||
static_assert(cute::rank(CtaTileMNK{}) == 3, "CtaTileMNK must be rank-3: [CTA_M, CTA_N, CTA_K]");
|
||||
static_assert(cute::rank(EpilogueTile{}) == 2, "EpilogueTile must be rank-2: [EPI_TILE_M, EPI_TILE_N]");
|
||||
static_assert(size<0>(CtaTileMNK{}) % size<0>(shape(EpilogueTile{})) == 0, "EPI_TILE_M must divide CTA_M");
|
||||
static_assert(size<1>(CtaTileMNK{}) % size<1>(shape(EpilogueTile{})) == 0, "EPI_TILE_N must divide CTA_N");
|
||||
static_assert(rank(StrideC{}) == 3, "StrideC must be rank-3: [M, N, L]");
|
||||
static_assert(rank(StrideD{}) == 3, "StrideD must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideC{}) == 3, "StrideC must be rank-3: [M, N, L]");
|
||||
static_assert(cute::rank(StrideD{}) == 3, "StrideD must be rank-3: [M, N, L]");
|
||||
|
||||
private:
|
||||
using SmemElementC = cute::conditional_t<cute::is_void_v<ElementC>,ElementD,ElementC>; // prevents void ref breakages
|
||||
@@ -340,10 +340,10 @@ public:
|
||||
auto [m_coord, n_coord, k_coord, l_coord] = tile_coord_mnkl;
|
||||
|
||||
// Tile residue
|
||||
auto m_max_coord = unwrap(cute::transform(make_seq<rank<0>(tile_shape_MNK)>{}, [&](auto i) {
|
||||
auto m_max_coord = unwrap(cute::transform(make_seq<cute::rank<0>(tile_shape_MNK)>{}, [&](auto i) {
|
||||
return get<0,i>(problem_shape_mnkl) - get<0,i>(tile_shape_MNK) * get<0,i>(tile_coord_mnkl);
|
||||
}));
|
||||
auto n_max_coord = unwrap(cute::transform(make_seq<rank<1>(tile_shape_MNK)>{}, [&](auto i) {
|
||||
auto n_max_coord = unwrap(cute::transform(make_seq<cute::rank<1>(tile_shape_MNK)>{}, [&](auto i) {
|
||||
return get<1,i>(problem_shape_mnkl) - get<1,i>(tile_shape_MNK) * get<1,i>(tile_coord_mnkl);
|
||||
}));
|
||||
auto residue_mn = make_coord(m_max_coord, n_max_coord);
|
||||
@@ -456,11 +456,11 @@ public:
|
||||
using ElementCompute = cute::conditional_t<cute::is_void_v<ElementCompute_>,ElementAccumulator,ElementCompute_>;
|
||||
|
||||
static_assert(is_rmem<AccEngine>::value, "Accumulator must be RF resident.");
|
||||
static_assert(rank(AccLayout{}) == 3, "Accumulator must be MMA-partitioned: (MMA,MMA_M,MMA_N)");
|
||||
static_assert(rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(cute::rank(AccLayout{}) == 3, "Accumulator must be MMA-partitioned: (MMA,MMA_M,MMA_N)");
|
||||
static_assert(cute::rank(ProblemShapeMNKL{}) == 4, "ProblemShapeMNKL must be rank 4");
|
||||
static_assert(is_static<TileShapeMNK>::value, "TileShapeMNK must be static");
|
||||
static_assert(rank(TileShapeMNK{}) == 3, "TileShapeMNK must be rank 3");
|
||||
static_assert(rank(TileCoordMNKL{}) == 4, "TileCoordMNKL must be rank 4");
|
||||
static_assert(cute::rank(TileShapeMNK{}) == 3, "TileShapeMNK must be rank 3");
|
||||
static_assert(cute::rank(TileCoordMNKL{}) == 4, "TileCoordMNKL must be rank 4");
|
||||
|
||||
// Indexing variables
|
||||
auto [M, N, K, L] = problem_shape_mnkl;
|
||||
@@ -530,11 +530,11 @@ public:
|
||||
Tensor bSG_gD = thrblk_s2g.partition_D(gD_epi); // (S2G,S2G_M,S2G_N,EPI_M,EPI_N)
|
||||
|
||||
// Coordinate tensors and residue for tile quantization
|
||||
auto m_max_coord = unwrap(cute::transform(make_seq<rank<0>(CtaTileMNK{})>{}, [&](auto i) {
|
||||
auto m_max_coord = unwrap(cute::transform(make_seq<cute::rank<0>(CtaTileMNK{})>{}, [&](auto i) {
|
||||
auto c_m = get<0,i>(problem_shape_mnkl) - get<0,i>(CtaTileMNK{}) * get<0,i>(tile_coord_mnkl);
|
||||
return cute::max(0, c_m);
|
||||
}));
|
||||
auto n_max_coord = unwrap(cute::transform(make_seq<rank<1>(CtaTileMNK{})>{}, [&](auto i) {
|
||||
auto n_max_coord = unwrap(cute::transform(make_seq<cute::rank<1>(CtaTileMNK{})>{}, [&](auto i) {
|
||||
auto c_n = get<1,i>(problem_shape_mnkl) - get<1,i>(CtaTileMNK{}) * get<1,i>(tile_coord_mnkl);
|
||||
return cute::max(0, c_n);
|
||||
}));
|
||||
@@ -559,7 +559,7 @@ public:
|
||||
tRS_cD,
|
||||
tRS_rC
|
||||
};
|
||||
auto cst_callbacks = fusion_callbacks.get_consumer_store_callbacks<RefSrc>(cst_args);
|
||||
auto cst_callbacks = fusion_callbacks.template get_consumer_store_callbacks<RefSrc>(cst_args);
|
||||
bool is_producer_load_needed = fusion_callbacks.is_producer_load_needed();
|
||||
bool is_C_load_needed = is_source_supported && fusion_callbacks.is_C_load_needed();
|
||||
|
||||
|
||||
@@ -695,7 +695,7 @@ template<
|
||||
FloatRoundStyle RoundStyle = FloatRoundStyle::round_to_nearest
|
||||
>
|
||||
using Sm90ScaledLinCombPerRowBiasEltAct =
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementOutput>::Op, ElementOutput, ElementCompute, RoundStyle>, // activation(Z) * scale_d
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementOutput>::template Op, ElementOutput, ElementCompute, RoundStyle>, // activation(Z) * scale_d
|
||||
Sm90EVT<Sm90Compute<ActivationFn, ElementCompute, ElementCompute, RoundStyle>, // activation(Z)
|
||||
// Z = scale_a * scale_b * alpha * acc + beta * scale_c * C + per-row bias
|
||||
Sm90ScaledLinCombPerRowBias<CtaTileShapeMNK, ElementCompute, ElementCompute, ElementBias, ElementScalar, AlignmentBias, RoundStyle>
|
||||
@@ -829,7 +829,7 @@ using Sm90ScaledLinCombPerRowBiasEltActAmaxAux =
|
||||
// Z = scale_a * scale_b * alpha * acc + scale_c * beta * C + per-row bias
|
||||
Sm90ScaledLinCombPerRowBias<CtaTileShapeMNK, ElementCompute, ElementCompute, ElementBias, ElementScalar, AlignmentBias, RoundStyle>,
|
||||
// D = activation(Z) * scale_d, amax_d = max(abs(elements in D))
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementOutput>::Op, ElementOutput, ElementCompute, RoundStyle>, // activation(Z) * scale_d
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementOutput>::template Op, ElementOutput, ElementCompute, RoundStyle>, // activation(Z) * scale_d
|
||||
Sm90EVT<Sm90ScalarReduction<detail::amax, atomic_maximum, ElementAmax, ElementCompute, RoundStyle>, // amax_d
|
||||
Sm90EVT<Sm90Compute<ActivationFn, ElementCompute, ElementCompute, RoundStyle>, // activation(Z)
|
||||
Sm90SplitTreeFetch // Z
|
||||
@@ -839,7 +839,7 @@ using Sm90ScaledLinCombPerRowBiasEltActAmaxAux =
|
||||
>,
|
||||
// Aux = Z * scale_aux, amax_aux = max(abs(elements in Aux))
|
||||
Sm90EVT<Sm90AuxStore<StagesD, EpilogueTile, ElementAux, RoundStyle, StrideAux, SmemLayoutAtom, CopyOpR2S, AlignmentAux>, // store(Aux)
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementAux>::Op, ElementCompute, ElementCompute, RoundStyle>, // Z * scale_aux
|
||||
Sm90EVT<Sm90Compute<detail::ScaleOutOp<ElementAux>::template Op, ElementCompute, ElementCompute, RoundStyle>, // Z * scale_aux
|
||||
Sm90EVT<Sm90ScalarReduction<detail::amax, atomic_maximum, ElementAmax, ElementCompute, RoundStyle>, // amax_aux
|
||||
Sm90SplitTreeFetch // Z
|
||||
>,
|
||||
@@ -1021,7 +1021,7 @@ template<
|
||||
using Sm90LinCombDeEltAct =
|
||||
Sm90EVT<Sm90Compute<ActivationFn, ElementOutput, ElementCompute, RoundStyle>, // activation(beta * C + (alpha * acc), aux)
|
||||
Sm90LinearCombination<ElementCompute, ElementCompute, ElementScalar, RoundStyle>, // beta * C + (alpha * acc)
|
||||
Sm90AuxLoad<Stages, EpilogueTile, ElementAux, StrideAux, SmemLayoutAtom, CopyOpS2R, AlignmentAux>, // aux
|
||||
Sm90AuxLoad<Stages, EpilogueTile, ElementAux, StrideAux, SmemLayoutAtom, CopyOpS2R, AlignmentAux> // aux
|
||||
>;
|
||||
|
||||
template <
|
||||
|
||||
@@ -237,6 +237,18 @@ struct Sm90TreeVisitor<
|
||||
Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>
|
||||
>;
|
||||
|
||||
using Params = typename Impl::Params;
|
||||
using SharedStorage = typename Impl::SharedStorage;
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor() {}
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor(
|
||||
Params const& params,
|
||||
SharedStorage const& shared_storage)
|
||||
: Impl(params, shared_storage) {}
|
||||
|
||||
CUTLASS_DEVICE bool
|
||||
is_producer_load_needed() const {
|
||||
auto const& bcast_op = get<0>(Impl::ops);
|
||||
@@ -252,8 +264,6 @@ struct Sm90TreeVisitor<
|
||||
return bcast_op.scalar != 0 || added_op.is_C_load_needed();
|
||||
}
|
||||
|
||||
using Impl::Sm90VisitorImpl;
|
||||
|
||||
template <class CallbacksImpl>
|
||||
struct ConsumerStoreCallbacks : CallbacksImpl {
|
||||
CUTLASS_DEVICE
|
||||
@@ -301,10 +311,9 @@ struct Sm90TreeVisitor<
|
||||
>
|
||||
CUTLASS_DEVICE auto
|
||||
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
|
||||
return ConsumerStoreCallbacks(
|
||||
is_C_load_needed(),
|
||||
Impl::get_consumer_store_callbacks<ReferenceSrc>(args)
|
||||
);
|
||||
auto callbacks_tuple = Impl::template get_consumer_store_callbacks<ReferenceSrc>(args);
|
||||
return ConsumerStoreCallbacks<decltype(callbacks_tuple)>(
|
||||
is_C_load_needed(), std::move(callbacks_tuple));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -475,7 +484,8 @@ struct Sm90ReLUAuxStore {
|
||||
gAux, args.epi_tile, args.tiled_copy, args.thread_idx);
|
||||
Tensor tC_rAux = make_tensor<cutlass::uint1b_t>(shape(tC_gAux)); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
|
||||
|
||||
return ConsumerStoreCallbacks(cute::move(tC_rAux), cute::move(tC_gAux), args.tCcD, args.residue_mn, params);
|
||||
return ConsumerStoreCallbacks<decltype(tC_rAux), decltype(tC_gAux), decltype(args.tCcD), decltype(args.residue_mn)>(
|
||||
cute::move(tC_rAux), cute::move(tC_gAux), args.tCcD, args.residue_mn, params);
|
||||
}
|
||||
};
|
||||
} // namespace detail
|
||||
@@ -532,7 +542,17 @@ struct Sm90TreeVisitor<
|
||||
Sm90Compute<Activation, ElementOutput, ElementCompute, RoundStyle>
|
||||
>;
|
||||
|
||||
using Impl::Sm90VisitorImpl;
|
||||
using Params = typename Impl::Params;
|
||||
using SharedStorage = typename Impl::SharedStorage;
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor() {}
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor(
|
||||
Params const& params,
|
||||
SharedStorage const& shared_storage)
|
||||
: Impl(params, shared_storage) {}
|
||||
|
||||
template <class CallbacksImpl>
|
||||
struct ConsumerStoreCallbacks : CallbacksImpl {
|
||||
@@ -556,9 +576,8 @@ struct Sm90TreeVisitor<
|
||||
>
|
||||
CUTLASS_DEVICE auto
|
||||
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
|
||||
return ConsumerStoreCallbacks(
|
||||
Impl::get_consumer_store_callbacks<ReferenceSrc>(args)
|
||||
);
|
||||
auto callbacks_tuple = Impl::template get_consumer_store_callbacks<ReferenceSrc>(args);
|
||||
return ConsumerStoreCallbacks<decltype(callbacks_tuple)>(std::move(callbacks_tuple));
|
||||
}
|
||||
|
||||
};
|
||||
@@ -654,7 +673,7 @@ struct Sm90AuxLoad<
|
||||
|
||||
CUTLASS_DEVICE void
|
||||
begin() {
|
||||
if constexpr (decltype(rank(tC_rAux))::value == 5) {
|
||||
if constexpr (decltype(cute::rank(tC_rAux))::value == 5) {
|
||||
if constexpr (EnableNullptr) {
|
||||
if (params.ptr_aux == nullptr) {
|
||||
return;
|
||||
@@ -669,7 +688,7 @@ struct Sm90AuxLoad<
|
||||
|
||||
CUTLASS_DEVICE void
|
||||
previsit(int epi_m, int epi_n, int load_iteration, bool is_producer_load_needed) {
|
||||
if constexpr (decltype(rank(tC_rAux))::value == 3) {
|
||||
if constexpr (decltype(cute::rank(tC_rAux))::value == 3) {
|
||||
if constexpr (EnableNullptr) {
|
||||
if (params.ptr_aux == nullptr) {
|
||||
return;
|
||||
@@ -686,7 +705,7 @@ struct Sm90AuxLoad<
|
||||
CUTLASS_DEVICE auto
|
||||
visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) {
|
||||
using ElementRegister = typename remove_cvref_t<RTensor>::value_type;
|
||||
if constexpr (decltype(rank(tC_rAux))::value == 3) {
|
||||
if constexpr (decltype(cute::rank(tC_rAux))::value == 3) {
|
||||
return recast<Array<ElementRegister, FragmentSize>>(coalesce(tC_rAux))(epi_v);
|
||||
}
|
||||
else {
|
||||
@@ -727,7 +746,8 @@ struct Sm90AuxLoad<
|
||||
}
|
||||
}
|
||||
|
||||
return ConsumerStoreCallbacks(cute::move(tC_rAux), cute::move(tC_gAux), args.residue_mn, params);
|
||||
return ConsumerStoreCallbacks<decltype(tC_rAux), decltype(tC_gAux), decltype(args.residue_mn)>(
|
||||
cute::move(tC_rAux), cute::move(tC_gAux), args.residue_mn, params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -280,7 +280,8 @@ struct Sm90AuxLoad {
|
||||
Tensor bGS_gAux = thrblk_g2s.partition_S(gAux_epi); // (TMA,TMA_M,TMA_N,EPI_M,EPI_N)
|
||||
Tensor bGS_sAux = thrblk_g2s.partition_D(sAux_epi); // (TMA,TMA_M,TMA_N,PIPE)
|
||||
|
||||
return ProducerLoadCallbacks(cute::move(bGS_gAux), cute::move(bGS_sAux), params_ptr);
|
||||
return ProducerLoadCallbacks<decltype(bGS_gAux), decltype(bGS_sAux)>(
|
||||
cute::move(bGS_gAux), cute::move(bGS_sAux), params_ptr);
|
||||
}
|
||||
|
||||
template <class RTensor, class TiledS2R, class STensorS2R>
|
||||
@@ -344,7 +345,8 @@ struct Sm90AuxLoad {
|
||||
auto tSR_sAux = tiled_s2r.get_slice(args.thread_idx).partition_S(sAux_epi); // (S2R,S2R_M,S2R_N,PIPE)
|
||||
|
||||
|
||||
return ConsumerStoreCallbacks(cute::move(tC_rAux), tiled_s2r, cute::move(tSR_sAux), params_ptr);
|
||||
return ConsumerStoreCallbacks<decltype(tC_rAux), decltype(tiled_s2r), decltype(tSR_sAux)>(
|
||||
cute::move(tC_rAux), tiled_s2r, cute::move(tSR_sAux), params_ptr);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -268,7 +268,7 @@ struct Sm90AuxStore {
|
||||
Tensor bSG_sAux = thrblk_s2g.partition_S(sAux_epi); // (TMA,TMA_M,TMA_N,PIPE)
|
||||
Tensor bSG_gAux = thrblk_s2g.partition_D(gAux_epi); // (TMA,TMA_M,TMA_N,EPI_M,EPI_N)
|
||||
|
||||
return ConsumerStoreCallbacks(
|
||||
return ConsumerStoreCallbacks<decltype(tC_rAux), decltype(tiled_r2s), decltype(tRS_sAux), decltype(bSG_sAux), decltype(bSG_gAux)>(
|
||||
cute::move(tC_rAux),
|
||||
tiled_r2s,
|
||||
cute::move(tRS_sAux),
|
||||
@@ -1109,12 +1109,11 @@ public:
|
||||
Tensor gBuf_nl = local_tile(mBuf, take<0,2>(args.tile_shape_mnk), make_coord(m,_,_)); // (CTA_M,CTA_N,REST_N,L)
|
||||
Layout sBuf_layout = blocked_product(gBuf_layout,make_layout(make_shape(_1{},_1{},size<1>(warp_layout_MN)))); // (CTA_M,CTA_N,WARPS_N)
|
||||
|
||||
return ConsumerStoreCallbacks(
|
||||
make_tuple(bool_constant<ReferenceSrc>{}, cute::move(tCrCol), args.tCcD, gCol_l, args.cD, gBuf_nl, sBuf_layout,
|
||||
lane_layout_MN, lane_mn, warp_layout_MN, warp_mn,
|
||||
args.tile_coord_mnkl, args.residue_mn, args.epi_tile, args.tiled_copy, args.thread_idx),
|
||||
params
|
||||
);
|
||||
auto args_tuple = make_tuple(
|
||||
bool_constant<ReferenceSrc>{}, cute::move(tCrCol), args.tCcD, gCol_l, args.cD, gBuf_nl, sBuf_layout,
|
||||
lane_layout_MN, lane_mn, warp_layout_MN, warp_mn,
|
||||
args.tile_coord_mnkl, args.residue_mn, args.epi_tile, args.tiled_copy, args.thread_idx);
|
||||
return ConsumerStoreCallbacks<decltype(args_tuple)>(std::move(args_tuple), params);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -272,8 +272,18 @@ struct Sm90VisitorImplBase {
|
||||
template <class... Ops>
|
||||
struct Sm90VisitorImpl : Sm90VisitorImplBase<Ops...> {
|
||||
|
||||
using Sm90VisitorImplBase<Ops...>::Sm90VisitorImplBase;
|
||||
using Sm90VisitorImplBase<Ops...>::ops;
|
||||
using Impl = Sm90VisitorImplBase<Ops...>;
|
||||
using Params = typename Impl::Params;
|
||||
using SharedStorage = typename Impl::SharedStorage;
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90VisitorImpl() {}
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90VisitorImpl(Params const& params, SharedStorage const& shared_storage)
|
||||
: Impl(params, shared_storage) {}
|
||||
|
||||
using Impl::ops;
|
||||
|
||||
//
|
||||
// Queries for kernel runtime
|
||||
@@ -506,7 +516,18 @@ using namespace detail;
|
||||
template <class NodeOp, class... ChildOps>
|
||||
struct Sm90TreeVisitor : Sm90VisitorImpl<ChildOps..., NodeOp> {
|
||||
|
||||
using Sm90VisitorImpl<ChildOps..., NodeOp>::Sm90VisitorImpl;
|
||||
using Impl = Sm90VisitorImpl<ChildOps..., NodeOp>;
|
||||
using Params = typename Impl::Params;
|
||||
using SharedStorage = typename Impl::SharedStorage;
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor() {}
|
||||
|
||||
CUTLASS_HOST_DEVICE
|
||||
Sm90TreeVisitor(
|
||||
Params const& params,
|
||||
SharedStorage const& shared_storage)
|
||||
: Impl(params, shared_storage) {}
|
||||
|
||||
template<class CallbacksImpl>
|
||||
struct ConsumerStoreCallbacks : CallbacksImpl {
|
||||
@@ -538,10 +559,9 @@ struct Sm90TreeVisitor : Sm90VisitorImpl<ChildOps..., NodeOp> {
|
||||
>
|
||||
CUTLASS_DEVICE auto
|
||||
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
|
||||
return ConsumerStoreCallbacks(
|
||||
Sm90VisitorImpl<ChildOps..., NodeOp>::
|
||||
get_consumer_store_callbacks<ReferenceSrc>(args)
|
||||
);
|
||||
auto callbacks_tuple = Sm90VisitorImpl<ChildOps..., NodeOp>::
|
||||
template get_consumer_store_callbacks<ReferenceSrc>(args);
|
||||
return ConsumerStoreCallbacks<decltype(callbacks_tuple)>(std::move(callbacks_tuple));
|
||||
}
|
||||
|
||||
};
|
||||
@@ -590,10 +610,9 @@ struct Sm90SplitTreeVisitor : Sm90VisitorImpl<InputTree, AuxOutTrees..., OutputT
|
||||
>
|
||||
CUTLASS_DEVICE auto
|
||||
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
|
||||
return ConsumerStoreCallbacks(
|
||||
Sm90VisitorImpl<InputTree, AuxOutTrees..., OutputTree>::
|
||||
get_consumer_store_callbacks<ReferenceSrc>(args)
|
||||
);
|
||||
auto callbacks_tuple = Sm90VisitorImpl<InputTree, AuxOutTrees..., OutputTree>::
|
||||
template get_consumer_store_callbacks<ReferenceSrc>(args);
|
||||
return ConsumerStoreCallbacks<decltype(callbacks_tuple)>(std::move(callbacks_tuple));
|
||||
}
|
||||
|
||||
};
|
||||
@@ -609,7 +628,7 @@ template<
|
||||
>
|
||||
struct Sm90TopologicalVisitor : Sm90VisitorImpl<Ops...> {
|
||||
static_assert(is_static_v<EdgeTuple>);
|
||||
static_assert(rank(EdgeTuple{}) == sizeof...(Ops));
|
||||
static_assert(cute::rank(EdgeTuple{}) == sizeof...(Ops));
|
||||
static_assert(sizeof...(Ops) > 1);
|
||||
|
||||
using Sm90VisitorImpl<Ops...>::Sm90VisitorImpl;
|
||||
@@ -669,10 +688,9 @@ struct Sm90TopologicalVisitor : Sm90VisitorImpl<Ops...> {
|
||||
>
|
||||
CUTLASS_DEVICE auto
|
||||
get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) {
|
||||
return ConsumerStoreCallbacks(
|
||||
Sm90VisitorImpl<Ops...>::
|
||||
get_consumer_store_callbacks<ReferenceSrc>(args)
|
||||
);
|
||||
auto callbacks_tuple = Sm90VisitorImpl<Ops...>::
|
||||
template get_consumer_store_callbacks<ReferenceSrc>(args);
|
||||
return ConsumerStoreCallbacks<decltype(callbacks_tuple)>(std::move(callbacks_tuple));
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
@@ -232,7 +232,7 @@ template<
|
||||
>
|
||||
struct TopologicalVisitor2x : VisitorImpl2x<Ops...> {
|
||||
static_assert(is_static_v<EdgeTuple>);
|
||||
static_assert(rank(EdgeTuple{}) == sizeof...(Ops));
|
||||
static_assert(cute::rank(EdgeTuple{}) == sizeof...(Ops));
|
||||
static_assert(sizeof...(Ops) > 1);
|
||||
|
||||
using VisitorImpl2x<Ops...>::VisitorImpl2x;
|
||||
|
||||
Reference in New Issue
Block a user