3.6.0 update (#2005)
* 3.6.0 update * doc and swap stuff --------- Co-authored-by: yuzhai <yuzhai@nvidia.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
co-authored by
yuzhai
Haicheng Wu
parent
e1cd8c7866
commit
3d261a5974
@@ -100,16 +100,16 @@ struct Copy_Atom<Copy_Traits<Args...>, CopyInternalType>
|
||||
if constexpr (is_constant<NumValSrc, decltype(size(src))>::value ||
|
||||
is_constant<NumValDst, decltype(size(dst))>::value) {
|
||||
// Dispatch to unpack to execute instruction
|
||||
return copy_unpack(*this, src, dst);
|
||||
} else
|
||||
if constexpr (is_tuple<decltype(shape(src))>::value &&
|
||||
is_tuple<decltype(shape(dst))>::value) {
|
||||
return copy_unpack(static_cast<Traits const&>(*this), src, dst);
|
||||
} else if constexpr (is_tuple<decltype(shape(src))>::value &&
|
||||
is_tuple<decltype(shape(dst))>::value) {
|
||||
// If the size of the src/dst doesn't match the instruction,
|
||||
// recurse this rank-1 layout by peeling off the mode
|
||||
// ((A,B,C,...)) -> (A,B,C,...)
|
||||
return copy(*this, tensor<0>(src), tensor<0>(dst));
|
||||
} else {
|
||||
static_assert(dependent_false<SEngine>, "No instruction match and no recursion possible.");
|
||||
static_assert(dependent_false<SEngine>,
|
||||
"CopyAtom: Src/Dst partitioning does not match the instruction requirement.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -92,23 +92,29 @@ struct Copy_Traits<AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>>
|
||||
using RefLayout = SrcLayout;
|
||||
};
|
||||
|
||||
// Extract a CPY_Op from a CPY_Traits
|
||||
template <class CPY_Traits>
|
||||
struct CPY_Op {};
|
||||
|
||||
template <class CPY_Op_Arg, class... Args>
|
||||
struct CPY_Op<Copy_Traits<CPY_Op_Arg, Args...>> {
|
||||
using type = CPY_Op_Arg;
|
||||
};
|
||||
|
||||
//
|
||||
// Generic copy_unpack for common argument-based Copy_Traits
|
||||
//
|
||||
|
||||
template <class CopyOp, class... Args,
|
||||
template <class AnyCPYTraits,
|
||||
class SEngine, class SLayout,
|
||||
class DEngine, class DLayout>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_unpack(Copy_Traits<CopyOp,Args...> const&,
|
||||
Tensor<SEngine,SLayout> const& src,
|
||||
Tensor<DEngine,DLayout> & dst)
|
||||
copy_unpack(AnyCPYTraits const&,
|
||||
Tensor<SEngine,SLayout> const& src,
|
||||
Tensor<DEngine,DLayout> & dst)
|
||||
{
|
||||
// Specializations can generalize on these checks
|
||||
//static_assert(is_smem<TS>::value, "Expected smem for this Copy_Traits<CopyOp>");
|
||||
//static_assert(is_rmem<TD>::value, "Expected rmem for this Copy_Traits<CopyOp>");
|
||||
|
||||
using CopyOp = typename CPY_Op<AnyCPYTraits>::type;
|
||||
using RegistersSrc = typename CopyOp::SRegisters;
|
||||
using RegistersDst = typename CopyOp::DRegisters;
|
||||
using RegTypeSrc = typename remove_extent<RegistersSrc>::type;
|
||||
@@ -129,18 +135,15 @@ copy_unpack(Copy_Traits<CopyOp,Args...> const&,
|
||||
rD, make_int_sequence<RegNumDst>{});
|
||||
}
|
||||
|
||||
//
|
||||
// Accept mutable temporaries
|
||||
//
|
||||
|
||||
template <class CopyOp, class... Args,
|
||||
template <class AnyCPYTraits,
|
||||
class SEngine, class SLayout,
|
||||
class DEngine, class DLayout>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_unpack(Copy_Traits<CopyOp,Args...> const& traits,
|
||||
Tensor<SEngine,SLayout> const& src,
|
||||
Tensor<DEngine,DLayout> && dst)
|
||||
copy_unpack(AnyCPYTraits const& traits,
|
||||
Tensor<SEngine,SLayout> const& src,
|
||||
Tensor<DEngine,DLayout> && dst)
|
||||
{
|
||||
copy_unpack(traits, src, dst);
|
||||
}
|
||||
|
||||
@@ -51,13 +51,6 @@ struct Copy_Traits<SM80_CP_ASYNC_CACHEALWAYS<S,D>>
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// Construct a zfill variant with a given predicate value
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM80_CP_ASYNC_CACHEALWAYS_ZFILL<S,D>>
|
||||
with(bool pred) const {
|
||||
return {pred};
|
||||
}
|
||||
};
|
||||
|
||||
template <class S, class D>
|
||||
@@ -73,13 +66,6 @@ struct Copy_Traits<SM80_CP_ASYNC_CACHEGLOBAL<S,D>>
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// Construct a zfill variant with a given predicate value
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<S,D>>
|
||||
with(bool pred) const {
|
||||
return {pred};
|
||||
}
|
||||
};
|
||||
|
||||
template <class S, class D>
|
||||
@@ -96,8 +82,15 @@ struct Copy_Traits<SM80_CP_ASYNC_CACHEALWAYS_ZFILL<S,D>>
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// Predicate value that determines whether to load or zfill
|
||||
bool pred = false;
|
||||
// Predicate value: true = load, false = zfill
|
||||
bool pred = true;
|
||||
|
||||
// Construct a zfill variant with a given predicate value
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM80_CP_ASYNC_CACHEALWAYS_ZFILL<S,D>>
|
||||
with(bool pred) const {
|
||||
return {pred};
|
||||
}
|
||||
|
||||
// Overload copy_unpack for zfill variant to pass the predicate into the op
|
||||
template <class TS, class SLayout,
|
||||
@@ -137,8 +130,15 @@ struct Copy_Traits<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<S,D>>
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// Predicate value that determines whether to load or zfill
|
||||
bool pred = false;
|
||||
// Predicate value: true = load, false = zfill
|
||||
bool pred = true;
|
||||
|
||||
// Construct a zfill variant with a given predicate value
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<S,D>>
|
||||
with(bool pred) const {
|
||||
return {pred};
|
||||
}
|
||||
|
||||
// Overload copy_unpack for zfill variant to pass the predicate into the op
|
||||
template <class TS, class SLayout,
|
||||
@@ -164,31 +164,4 @@ struct Copy_Traits<SM80_CP_ASYNC_CACHEGLOBAL_ZFILL<S,D>>
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Element copy selector
|
||||
template <class SrcTensor, class DstTensor>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
select_elementwise_copy(SrcTensor const&, DstTensor const&)
|
||||
{
|
||||
using SrcType = typename SrcTensor::value_type;
|
||||
using DstType = typename DstTensor::value_type;
|
||||
|
||||
#if defined(CUTE_ARCH_CP_ASYNC_SM80_ENABLED)
|
||||
if constexpr (is_gmem<SrcTensor>::value && is_smem<DstTensor>::value &&
|
||||
sizeof(SrcType) == sizeof(DstType) &&
|
||||
(sizeof(SrcType) == 4 || sizeof(SrcType) == 8 || sizeof(SrcType) == 16))
|
||||
{
|
||||
return SM80_CP_ASYNC_CACHEALWAYS<SrcType,DstType>{};
|
||||
} else {
|
||||
return UniversalCopy<SrcType,DstType>{};
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
#else
|
||||
return UniversalCopy<SrcType,DstType>{};
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
} // end namespace cute
|
||||
|
||||
@@ -58,37 +58,31 @@ struct AuxTmaParams {
|
||||
};
|
||||
|
||||
// Utility for unpacking TMA_LOAD arguments into a CopyOp
|
||||
template <class CopyOp>
|
||||
template <class CopyOp, class... Args>
|
||||
struct TMA_LOAD_Unpack
|
||||
{
|
||||
template <class... Args,
|
||||
class TS, class SLayout,
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits<CopyOp, Args...> const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TD>::value, "SM90_TMA_LOAD requires the destination be shared memory.");
|
||||
|
||||
auto src_coord = src.data().coord_;
|
||||
if constexpr (detail::is_prefetch<CopyOp>) {
|
||||
return detail::explode_tuple(detail::CallCOPY<CopyOp>{},
|
||||
traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
} else {
|
||||
static_assert(is_smem<TD>::value, "SM90_TMA_LOAD requires the destination be shared memory.");
|
||||
void* dst_ptr = cute::raw_pointer_cast(dst.data());
|
||||
void* dst_ptr = cute::raw_pointer_cast(dst.data());
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(src_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), dst_ptr);
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(src_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), dst_ptr);
|
||||
#endif
|
||||
return detail::explode_tuple(detail::CallCOPY<CopyOp>{},
|
||||
traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
make_tuple(dst_ptr), seq<0>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
}
|
||||
return detail::explode_tuple(detail::CallCOPY<CopyOp>{},
|
||||
traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
make_tuple(dst_ptr), seq<0>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -131,7 +125,7 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
[[maybe_unused]] uint16_t const& multicast_mask = 0,
|
||||
TMA::CacheHintSm90 const& cache_hint = TMA::CacheHintSm90::EVICT_NORMAL) const {
|
||||
// We accept multicast_mask here to keep the API for both atoms consistent
|
||||
return {{}, {&tma_desc_, &tma_mbar, static_cast<uint64_t>(cache_hint)}};
|
||||
return {&tma_desc_, &tma_mbar, static_cast<uint64_t>(cache_hint)};
|
||||
}
|
||||
|
||||
// Construct an executable SM90_TMA_LOAD with tma_mbar (temp. overloaded for grouped gemm/ptr array gemm)
|
||||
@@ -143,7 +137,7 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
[[maybe_unused]] uint16_t const& multicast_mask = 0,
|
||||
TMA::CacheHintSm90 const& cache_hint = TMA::CacheHintSm90::EVICT_NORMAL) const {
|
||||
// We accept multicast_mask here to keep the API for both atoms consistent
|
||||
return {{}, {new_tma_desc, &tma_mbar, static_cast<uint64_t>(cache_hint)}};
|
||||
return {new_tma_desc, &tma_mbar, static_cast<uint64_t>(cache_hint)};
|
||||
}
|
||||
|
||||
// Generate the TMA coord tensor
|
||||
@@ -167,7 +161,7 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
// The executable SM90_TMA_LOAD with tma_desc and tma_mbar
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_OP>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
@@ -183,12 +177,15 @@ struct Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
uint64_t*, // smem mbarrier
|
||||
uint64_t // cache hint
|
||||
> const opargs_;
|
||||
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(TmaDescriptor const* desc, uint64_t* mbar, uint64_t cache)
|
||||
: opargs_(desc, mbar, cache) {}
|
||||
};
|
||||
|
||||
// The prefetch for SM90_TMA_LOAD with tma_desc
|
||||
template <class NumBitsPerTMA, class... Args>
|
||||
struct Copy_Traits<SM90_TMA_LOAD::PREFETCH, NumBitsPerTMA, Args...>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD::PREFETCH>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
@@ -206,6 +203,19 @@ struct Copy_Traits<SM90_TMA_LOAD::PREFETCH, NumBitsPerTMA, Args...>
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(Copy_Traits<CopyArgs...> const& traits)
|
||||
: opargs_({&traits.tma_desc_}) {}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
auto src_coord = src.data().coord_;
|
||||
return detail::explode_tuple(detail::CallCOPY<SM90_TMA_LOAD::PREFETCH>{},
|
||||
traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -246,7 +256,7 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
uint64_t& tma_load_mbar,
|
||||
uint16_t const& multicast_mask,
|
||||
TMA::CacheHintSm90 const& cache_hint = TMA::CacheHintSm90::EVICT_NORMAL) const {
|
||||
return {{}, {&tma_desc_, &tma_load_mbar, multicast_mask, static_cast<uint64_t>(cache_hint)}};
|
||||
return {&tma_desc_, &tma_load_mbar, multicast_mask, static_cast<uint64_t>(cache_hint)};
|
||||
}
|
||||
|
||||
// Construct an executable SM90_TMA_LOAD_MULTICAST_OP with tma_mbar (temp. overloaded for grouped gemm/ptr array gemm)
|
||||
@@ -257,7 +267,7 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
uint64_t& tma_load_mbar,
|
||||
uint16_t const& multicast_mask,
|
||||
TMA::CacheHintSm90 const& cache_hint = TMA::CacheHintSm90::EVICT_NORMAL) const {
|
||||
return {{}, {new_tma_desc, &tma_load_mbar, multicast_mask, static_cast<uint64_t>(cache_hint)}};
|
||||
return {new_tma_desc, &tma_load_mbar, multicast_mask, static_cast<uint64_t>(cache_hint)};
|
||||
}
|
||||
|
||||
// Generate the TMA coord tensor
|
||||
@@ -281,7 +291,7 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
// The executable SM90_TMA_LOAD_MULTICAST with tma_desc and tma_mbar and multicast_mask
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_MULTICAST_OP>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
@@ -298,43 +308,17 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
uint16_t, // multicast mask
|
||||
uint64_t // cache hint
|
||||
> const opargs_;
|
||||
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(TmaDescriptor const* desc, uint64_t* mbar, uint16_t mask, uint64_t hint)
|
||||
: opargs_(desc, mbar, mask, hint) {}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_STORE //////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Utility for unpacking TMA_STORE arguments into a CopyOp
|
||||
template <class CopyOp>
|
||||
struct TMA_STORE_Unpack
|
||||
{
|
||||
template <class... Args,
|
||||
class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits<CopyOp, Args...> const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_STORE");
|
||||
|
||||
void const* const desc_ptr = traits.tma_desc_;
|
||||
void const* const src_ptr = cute::raw_pointer_cast(src.data());
|
||||
auto dst_coord = dst.data().coord_;
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(dst_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), src_ptr);
|
||||
#endif
|
||||
return detail::explode_tuple(detail::CallCOPY<SM90_TMA_STORE>{},
|
||||
make_tuple(desc_ptr, src_ptr), seq<0,1>{},
|
||||
dst_coord, tuple_seq<decltype(dst_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
struct SM90_TMA_STORE_OP : SM90_TMA_STORE {};
|
||||
struct SM90_TMA_STORE_PTR : SM90_TMA_STORE {};
|
||||
|
||||
// The executable SM90_TMA_STORE with tma_desc
|
||||
template <class NumBitsPerTMA, class AuxParams_>
|
||||
@@ -369,6 +353,13 @@ struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
return make_counting_tensor(make_layout(g_shape, aux_params_.g_stride_));
|
||||
}
|
||||
|
||||
// Construct new TMA_STORE with (unsafe) swapped out TMA descriptor ptr (for grouped gemm/ptr array gemm)
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_STORE_PTR, NumBitsPerTMA>
|
||||
with(TmaDescriptor const* new_tma_desc) const {
|
||||
return {new_tma_desc};
|
||||
}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
@@ -393,19 +384,11 @@ struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
make_tuple(desc_ptr, src_ptr), seq<0,1>{},
|
||||
dst_coord, tuple_seq<decltype(dst_coord)>{});
|
||||
}
|
||||
|
||||
// Construct Copy_Traits executable (w/ swapped out TMA descriptor) for SM90_TMA_STORE (for grouped gemm/ptr array gemm)
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_STORE_OP, NumBitsPerTMA>
|
||||
with(TmaDescriptor const* new_tma_desc) const {
|
||||
return {{}, new_tma_desc};
|
||||
}
|
||||
};
|
||||
|
||||
// The executable SM90_TMA_STORE with tma_desc
|
||||
// Same as SM90_TMA_STORE, but with an unsafe TMA Desc PTR instead
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_STORE_OP, NumBitsPerTMA>
|
||||
: TMA_STORE_Unpack<SM90_TMA_STORE_OP>
|
||||
struct Copy_Traits<SM90_TMA_STORE_PTR, NumBitsPerTMA>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
@@ -417,6 +400,31 @@ struct Copy_Traits<SM90_TMA_STORE_OP, NumBitsPerTMA>
|
||||
|
||||
// SM90_TMA_STORE arguments
|
||||
TmaDescriptor const* tma_desc_;
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_STORE");
|
||||
//static_assert(is_gmem<TD>::value, "Expected gmem dst for SM90_TMA_STORE"); // TMA spoofed src tensor
|
||||
|
||||
void const* const desc_ptr = traits.tma_desc_;
|
||||
void const* const src_ptr = cute::raw_pointer_cast(src.data());
|
||||
auto dst_coord = dst.data().coord_;
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(dst_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), src_ptr);
|
||||
#endif
|
||||
return detail::explode_tuple(detail::CallCOPY<SM90_TMA_STORE_PTR>{},
|
||||
make_tuple(desc_ptr, src_ptr), seq<0,1>{},
|
||||
dst_coord, tuple_seq<decltype(dst_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -520,7 +528,7 @@ struct Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, OpArgs...>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, uint64_t*>
|
||||
with(uint64_t& bulk_mbar) const {
|
||||
return {{&bulk_mbar}};
|
||||
return {&bulk_mbar};
|
||||
}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
@@ -613,7 +621,7 @@ struct Copy_Traits<SM90_BULK_COPY_AUTO, OpArgs...>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_BULK_COPY_AUTO, uint64_t*>
|
||||
with(uint64_t& bulk_mbar) const {
|
||||
return {{&bulk_mbar}};
|
||||
return {&bulk_mbar};
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1391,19 +1399,46 @@ tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
return cute::make_tuple(gresult, sresult);
|
||||
}
|
||||
|
||||
// Explicit defaults for cta_coord and cta_layout
|
||||
template <class... Args,
|
||||
class SEngine, class SLayout,
|
||||
class GEngine, class GLayout>
|
||||
CUTE_DEVICE
|
||||
auto
|
||||
tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
Tensor<SEngine,SLayout> const& stensor, // SMEM Tensor (TMATile, Rest...)
|
||||
Tensor<GEngine,GLayout> const& gtensor) // GMEM Tensor (TMATile, Rest...)
|
||||
{
|
||||
return tma_partition(copy_atom, Int<0>{}, Layout<_1,_0>{}, stensor, gtensor);
|
||||
}
|
||||
|
||||
// TMA Multicast Masks Calculation
|
||||
template <int Mode, class CtaLayout, class CtaCoord>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
uint16_t
|
||||
create_tma_multicast_mask(CtaLayout const& cta_layout_vmnk,
|
||||
CtaCoord const& cta_coord_vmnk)
|
||||
{
|
||||
auto cta_coord_slicer = replace<Mode>(cta_coord_vmnk, _);
|
||||
auto [cta_layout, elected_cta] = slice_and_offset(cta_coord_slicer, cta_layout_vmnk);
|
||||
// Get the instruction code
|
||||
|
||||
uint16_t mcast_mask = 0;
|
||||
for (int i = 0; i < size(cta_layout); ++i) {
|
||||
mcast_mask |= uint16_t(1) << cta_layout(i);
|
||||
if constexpr (rank_v<decltype(cta_layout)> == 1 and depth_v<decltype(cta_layout)> <= 1 and
|
||||
not is_static<decltype(cta_layout)>::value) {
|
||||
// Get the instruction code -- optimized for dynamic flat-rank-1 cta_layout
|
||||
mcast_mask = uint16_t(1);
|
||||
// Smear by stride<0> (may want to predicate on stride<0> mag?)
|
||||
mcast_mask |= mcast_mask << (1*stride<0>(cta_layout));
|
||||
mcast_mask |= mcast_mask << (2*stride<0>(cta_layout));
|
||||
mcast_mask |= mcast_mask << (4*stride<0>(cta_layout));
|
||||
mcast_mask |= mcast_mask << (8*stride<0>(cta_layout));
|
||||
// Select shape<0>
|
||||
mcast_mask &= (uint16_t(-1) >> (16 - shape<0>(cta_layout) * stride<0>(cta_layout)));
|
||||
} else {
|
||||
// Get the instruction code -- generic path
|
||||
for (int i = 0; i < size(cta_layout); ++i) {
|
||||
mcast_mask |= uint16_t(1) << cta_layout(i);
|
||||
}
|
||||
}
|
||||
// Shift by the instruction's elected block rank (dynamic)
|
||||
mcast_mask <<= elected_cta;
|
||||
|
||||
@@ -250,12 +250,12 @@ struct TiledMMA : MMA_Atom
|
||||
auto t_tensor = logical_divide(ctensor, t_tile); // (PermM,PermN)
|
||||
|
||||
// Tile the tensor for the Atom
|
||||
auto a_tile = make_tile(make_layout(size<0>(AtomShape_MNK{})),
|
||||
auto c_tile = make_tile(make_layout(size<0>(AtomShape_MNK{})),
|
||||
make_layout(size<1>(AtomShape_MNK{})));
|
||||
auto a_tensor = zipped_divide(t_tensor, a_tile); // ((AtomM,AtomN),(RestM,RestN))
|
||||
auto c_tensor = zipped_divide(t_tensor, c_tile); // ((AtomM,AtomN),(RestM,RestN))
|
||||
|
||||
// Transform the Atom mode from (M,K) to (Thr,Val)
|
||||
auto tv_tensor = a_tensor.compose(AtomLayoutC_TV{},_); // ((ThrV,FrgV),(RestM,RestN))
|
||||
auto tv_tensor = c_tensor.compose(AtomLayoutC_TV{},_); // ((ThrV,FrgV),(RestM,RestN))
|
||||
|
||||
// Tile the tensor for the C-threads
|
||||
auto thr_tile = make_tile(_,
|
||||
@@ -604,16 +604,15 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
partition_shape_C(TiledMMA<Args...> const& mma, Shape_MN const& shape_MN)
|
||||
{
|
||||
constexpr int R = rank_v<Shape_MN>;
|
||||
static_assert(R >= 2, "Must have at least rank-2");
|
||||
auto atomMNK = typename TiledMMA<Args...>::AtomShape_MNK{};
|
||||
auto thrVMNK = typename TiledMMA<Args...>::ThrLayoutVMNK{};
|
||||
auto V = shape<1>(typename TiledMMA<Args...>::AtomLayoutC_TV{});
|
||||
auto M = shape_div(size<0>(shape_MN), size<0>(atomMNK) * size<1>(thrVMNK));
|
||||
auto N = shape_div(size<1>(shape_MN), size<1>(atomMNK) * size<2>(thrVMNK));
|
||||
return cute::tuple_cat(make_shape(V,M,N), take<2,R>(shape_MN));
|
||||
auto dummy = make_layout(shape(shape_MN));
|
||||
auto dummy_tv = mma.thrfrg_C(dummy);
|
||||
// Slice+rearrange like partition_C
|
||||
auto dummy_v = dummy_tv(Int<0>{}, make_coord(_, repeat<rank(dummy)>(_)));
|
||||
return shape(dummy_v);
|
||||
|
||||
}
|
||||
|
||||
|
||||
template <class... Args, class Shape_MN>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -632,14 +631,12 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
partition_shape_A(TiledMMA<Args...> const& mma, Shape_MK const& shape_MK)
|
||||
{
|
||||
constexpr int R = rank_v<Shape_MK>;
|
||||
static_assert(R >= 2, "Must have at least rank-2");
|
||||
auto atomMNK = typename TiledMMA<Args...>::AtomShape_MNK{};
|
||||
auto thrVMNK = typename TiledMMA<Args...>::ThrLayoutVMNK{};
|
||||
auto V = shape<1>(typename TiledMMA<Args...>::AtomLayoutA_TV{});
|
||||
auto M = shape_div(size<0>(shape_MK), size<0>(atomMNK) * size<1>(thrVMNK));
|
||||
auto K = shape_div(size<1>(shape_MK), size<2>(atomMNK) * size<3>(thrVMNK));
|
||||
return cute::tuple_cat(make_shape(V,M,K), take<2,R>(shape_MK));
|
||||
auto dummy = make_layout(shape(shape_MK));
|
||||
auto dummy_tv = mma.thrfrg_A(dummy);
|
||||
// Slice+rearrange like partition_A
|
||||
auto dummy_v = dummy_tv(Int<0>{}, make_coord(_, repeat<rank(dummy)>(_)));
|
||||
return shape(dummy_v);
|
||||
|
||||
}
|
||||
|
||||
template <class... Args, class Shape_NK>
|
||||
@@ -647,14 +644,12 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
partition_shape_B(TiledMMA<Args...> const& mma, Shape_NK const& shape_NK)
|
||||
{
|
||||
constexpr int R = rank_v<Shape_NK>;
|
||||
static_assert(R >= 2, "Must have at least rank-2");
|
||||
auto atomMNK = typename TiledMMA<Args...>::AtomShape_MNK{};
|
||||
auto thrVMNK = typename TiledMMA<Args...>::ThrLayoutVMNK{};
|
||||
auto V = shape<1>(typename TiledMMA<Args...>::AtomLayoutB_TV{});
|
||||
auto N = shape_div(size<0>(shape_NK), size<1>(atomMNK) * size<2>(thrVMNK));
|
||||
auto K = shape_div(size<1>(shape_NK), size<2>(atomMNK) * size<3>(thrVMNK));
|
||||
return cute::tuple_cat(make_shape(V,N,K), take<2,R>(shape_NK));
|
||||
auto dummy = make_layout(shape(shape_NK));
|
||||
auto dummy_tv = mma.thrfrg_B(dummy);
|
||||
// Slice+rearrange like partition_B
|
||||
auto dummy_v = dummy_tv(Int<0>{}, make_coord(_, repeat<rank(dummy)>(_)));
|
||||
return shape(dummy_v);
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
@@ -419,6 +419,203 @@ template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32U8U8S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x32_S32U8U8S32_TN> {};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = s4 * s4 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
|
||||
using Shape_MNK = Shape<_8, _8, _32>;
|
||||
using ThrID = Layout<_32>;
|
||||
// (T32,V8) -> (M8,N32)
|
||||
using ALayout = Layout<Shape <Shape < _4, _8>, Shape <_8>>,
|
||||
Stride<Stride<_64, _1>, Stride<_8>>>;
|
||||
using BLayout = Layout<Shape <Shape < _4, _8>, Shape <_8>>,
|
||||
Stride<Stride<_64, _1>, Stride<_8>>>;
|
||||
using CLayout = SM80_8x8_Row;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32S4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_8x8x32_S32S4S4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
|
||||
using Shape_MNK = Shape<_16, _8, _32>;
|
||||
using ThrID = Layout<_32>;
|
||||
// (T32,V16) -> (M16,N32)
|
||||
using ALayout = Layout<Shape <Shape < _4, _8>, Shape < _8, _2>>,
|
||||
Stride<Stride<_128, _1>, Stride<_16, _8>>>;
|
||||
// (T32,V8) -> (M8,N32)
|
||||
using BLayout = Layout<Shape <Shape < _4, _8>, Shape <_8>>,
|
||||
Stride<Stride<_32, _1>, Stride<_8>>>;
|
||||
using CLayout = SM80_16x8_Row;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32S4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x32_S32S4S4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
|
||||
using Shape_MNK = Shape<_16, _8, _64>;
|
||||
using ThrID = Layout<_32>;
|
||||
// (T32,V32) -> (M16,N64)
|
||||
using ALayout = Layout<Shape <Shape < _4, _8>, Shape < _8, _2, _2>>,
|
||||
Stride<Stride<_128, _1>, Stride<_16, _8, _512>>>;
|
||||
// (T32,V16) -> (M8,N64)
|
||||
using BLayout = Layout<Shape <Shape < _4, _8>, Shape <_8, _2>>,
|
||||
Stride<Stride<_64, _1>, Stride<_8, _256>>>;
|
||||
using CLayout = SM80_16x8_Row;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32S4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x64_S32S4S4S32_TN> {};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = s4 * u4 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32S4U4S32_TN>
|
||||
: MMA_Traits<SM80_8x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32S4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_8x8x32_S32S4U4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32S4U4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32S4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x32_S32S4U4S32_TN> {};
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32S4U4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x64_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = int4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32S4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x64_S32S4U4S32_TN> {};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = u4 * s4 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32U4S4S32_TN>
|
||||
: MMA_Traits<SM80_8x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32U4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_8x8x32_S32U4S4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32U4S4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32U4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x32_S32U4S4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32U4S4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x64_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = int4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32U4S4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x64_S32U4S4S32_TN> {};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = u4 * u4 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32U4U4S32_TN>
|
||||
: MMA_Traits<SM80_8x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_8x8x32_S32U4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_8x8x32_S32U4U4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32U4U4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x32_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x32_S32U4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x32_S32U4U4S32_TN> {};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32U4U4S32_TN>
|
||||
: MMA_Traits<SM80_16x8x64_S32S4S4S32_TN> {
|
||||
using ValTypeD = int32_t;
|
||||
using ValTypeA = uint4b_t;
|
||||
using ValTypeB = uint4b_t;
|
||||
using ValTypeC = int32_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x64_S32U4U4S32_TN_SATURATE>
|
||||
: MMA_Traits<SM80_16x8x64_S32U4U4S32_TN> {};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = b1 ^ b1 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
@@ -440,9 +637,13 @@ struct MMA_Traits<SM80_16x8x256_S32U1U1S32_TN_XORPOPC>
|
||||
using CLayout = SM80_16x8_Row;
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////// s32 = b1 & b1 + s32 ///////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <>
|
||||
struct MMA_Traits<SM80_16x8x256_S32U1U1S32_TN_ANDPOPC>
|
||||
:MMA_Traits<SM80_16x8x256_S32U1U1S32_TN_XORPOPC> {};
|
||||
: MMA_Traits<SM80_16x8x256_S32U1U1S32_TN_XORPOPC> {};
|
||||
|
||||
template<>
|
||||
struct MMA_Traits<SM80_8x8x128_S32U1U1S32_TN_XORPOPC>
|
||||
@@ -455,7 +656,7 @@ struct MMA_Traits<SM80_8x8x128_S32U1U1S32_TN_XORPOPC>
|
||||
using Shape_MNK = Shape<_8,_8,_128>;
|
||||
using ThrID = Layout<_32>;
|
||||
using ALayout = Layout<Shape<Shape<_4,_8>,_32>,
|
||||
Stride<Stride<_256,_1>,_8>>;
|
||||
Stride<Stride<_256,_1>,_8>>;
|
||||
using BLayout = Layout<Shape<Shape<_4,_8>,_32>,
|
||||
Stride<Stride<_256,_1>,_8>>;
|
||||
using CLayout = SM80_8x8_Row;
|
||||
@@ -472,7 +673,7 @@ struct MMA_Traits<SM80_16x8x128_S32U1U1S32_TN_XORPOPC>
|
||||
using ValTypeA = cute::uint1b_t;
|
||||
using ValTypeB = cute::uint1b_t;
|
||||
using ValTypeC = int32_t;
|
||||
|
||||
|
||||
using Shape_MNK = Shape<_16,_8,_128>;
|
||||
using ThrID = Layout<_32>;
|
||||
using ALayout = Layout<Shape<Shape<_4,_8>,Shape<_32,_2>>,
|
||||
|
||||
@@ -1128,7 +1128,6 @@ struct MMA_Traits<SM90_64x32x16_F32F16F16_RS<tnspA, tnspB, scaleA, scaleB>>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
template <
|
||||
GMMA::Major tnspA,
|
||||
GMMA::Major tnspB,
|
||||
|
||||
@@ -7735,4 +7735,4 @@ struct MMA_Traits<SM90::GMMA::SPARSE::GMMA_64x256x64_F32E5M2E5M2_RS_TN<scaleA, s
|
||||
|
||||
#if defined(CUTE_SM90_EXTENDED_MMA_SHAPES_ENABLED)
|
||||
#include "mma_traits_sm90_gmma_sparse_ext.hpp"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user