CUTLASS 3.5.0 (#1411)
This commit is contained in:
@@ -33,6 +33,7 @@
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/tensor_predicate.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -43,15 +44,17 @@ namespace cute
|
||||
template <class Alpha,
|
||||
class XEngine, class XLayout,
|
||||
class Beta,
|
||||
class YEngine, class YLayout>
|
||||
class YEngine, class YLayout,
|
||||
class PrdTensor = TrivialPredTensor>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
axpby(Alpha const& alpha,
|
||||
Tensor<XEngine, XLayout> const& x,
|
||||
Beta const& beta,
|
||||
Tensor<YEngine, YLayout> && y)
|
||||
Tensor<YEngine, YLayout> && y,
|
||||
PrdTensor const& p = {})
|
||||
{
|
||||
return axpby(alpha, x, beta, y);
|
||||
return axpby(alpha, x, beta, y, p);
|
||||
}
|
||||
|
||||
//
|
||||
@@ -60,13 +63,15 @@ axpby(Alpha const& alpha,
|
||||
template <class Alpha,
|
||||
class XEngine, class XLayout,
|
||||
class Beta,
|
||||
class YEngine, class YLayout>
|
||||
class YEngine, class YLayout,
|
||||
class PrdTensor = TrivialPredTensor>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
axpby(Alpha const& alpha,
|
||||
Tensor<XEngine, XLayout> const& x,
|
||||
Beta const& beta,
|
||||
Tensor<YEngine, YLayout> & y)
|
||||
Tensor<YEngine, YLayout> & y,
|
||||
PrdTensor const& p = {})
|
||||
{
|
||||
auto isBetaZero = [&] () {
|
||||
if constexpr (is_complex<Beta>::value) {
|
||||
@@ -81,7 +86,9 @@ axpby(Alpha const& alpha,
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(x); ++i) {
|
||||
y(i) = (isBetaZero ? alpha * x(i) : alpha * x(i) + beta * y(i));
|
||||
if (p(i)) {
|
||||
y(i) = (isBetaZero ? alpha * x(i) : alpha * x(i) + beta * y(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/atom/copy_atom.hpp>
|
||||
|
||||
#include <cute/algorithm/copy.hpp>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/tensor_predicate.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
// cooperative_copy<NumThreads, MaxVecBits>(thr_idx, src, dst)
|
||||
// Use NumThreads to copy src to dst with element vectorization up to MaxVecBits.
|
||||
// @pre 0 <= @a tid < NumThreads
|
||||
// @pre Tensors @a src and @a dst are aligned up to MaxVecBits.
|
||||
//
|
||||
template <uint32_t NumThreads, uint32_t MaxVecBits,
|
||||
class SrcEngine, class SrcLayout,
|
||||
class DstEngine, class DstLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_copy(uint32_t const& tid,
|
||||
Tensor<SrcEngine, SrcLayout> const& src,
|
||||
Tensor<DstEngine, DstLayout> & dst)
|
||||
{
|
||||
// Assumes the shapes are static, can generalize
|
||||
CUTE_STATIC_ASSERT_V(size(src) == size(dst));
|
||||
// Assumes the types are the same, can generalize
|
||||
static_assert(sizeof_bits_v<typename SrcEngine::value_type> == sizeof_bits_v<typename DstEngine::value_type>);
|
||||
static_assert(MaxVecBits == sizeof_bits_v<typename SrcEngine::value_type> ||
|
||||
MaxVecBits == 8 || MaxVecBits == 16 || MaxVecBits == 32 || MaxVecBits == 64 || MaxVecBits == 128,
|
||||
"Expected MaxVecBits to be value size or 8 or 16 or 32 or 64 or 128 for alignment and performance.");
|
||||
// Check that the tensors are likely shared across threads: either gmem or smem
|
||||
static_assert((is_gmem<SrcEngine>::value || is_smem<SrcEngine>::value),
|
||||
"cooperative_copy expects shared gmem or smem source tensor.");
|
||||
static_assert((is_gmem<DstEngine>::value || is_smem<DstEngine>::value),
|
||||
"cooperative_copy expects shared gmem or smem destination tensor.");
|
||||
|
||||
// Precondition on tid in DEBUG
|
||||
assert(tid < NumThreads);
|
||||
// Precondition on pointer alignment in DEBUG
|
||||
assert(is_byte_aligned<max(MaxVecBits/8, 1u)>(raw_pointer_cast(src.data())));
|
||||
assert(is_byte_aligned<max(MaxVecBits/8, 1u)>(raw_pointer_cast(dst.data())));
|
||||
//
|
||||
// Determine val+thr vectorization based on src/dst size and number of threads
|
||||
// NOTE: This heuristic promotes parallelization over vectorization
|
||||
//
|
||||
|
||||
constexpr int elem_bits = sizeof_bits_v<typename SrcEngine::value_type>;
|
||||
|
||||
// The number of elements that can be vectorized in values
|
||||
constexpr int common_elem = decltype(max_common_vector(src, dst))::value;
|
||||
constexpr int common_bits = common_elem * elem_bits;
|
||||
constexpr int total_elem = decltype(size(src))::value;
|
||||
constexpr int total_bits = total_elem * elem_bits;
|
||||
static_assert(total_bits % NumThreads == 0);
|
||||
constexpr int total_bits_per_thr = total_bits / NumThreads;
|
||||
// If there are too many threads to allow a full elem copy, trunc the thrs and use elem_bits
|
||||
constexpr int max_vec_bits_by_thr = cute::max(elem_bits, total_bits_per_thr);
|
||||
|
||||
// Cap the vectorization to the common bits, the max_vec_bits_by_thr, and the MaxVecBits
|
||||
constexpr int vec_bits = cute::min(common_bits, max_vec_bits_by_thr, static_cast<int>(MaxVecBits));
|
||||
// Convert back to number of elements, safe_div
|
||||
static_assert((vec_bits % elem_bits) == 0);
|
||||
constexpr int vec_elem = vec_bits / elem_bits;
|
||||
|
||||
// Use only part of threads if there's not enough work for all threads
|
||||
constexpr int vec_thrs = (total_elem % (vec_elem * NumThreads) == 0)
|
||||
? NumThreads
|
||||
: (total_elem / vec_elem);
|
||||
|
||||
// The common layout of the two tensors that can be vectorized over threads
|
||||
// vidx -> coord
|
||||
auto common_layout = max_common_layout(get_nonswizzle_portion(src.layout()),
|
||||
get_nonswizzle_portion(dst.layout()));
|
||||
|
||||
// Scale up the common_layout to cover the entire tensors
|
||||
// vidx -> coord
|
||||
auto full_perm = tile_to_shape(make_layout(common_layout), size(src));
|
||||
|
||||
// Create the Tiler
|
||||
// ((vid,tid),iter)
|
||||
auto layout_vt = logical_divide(full_perm, Layout<Shape<Int<vec_elem>, Int<vec_thrs>>>{});
|
||||
|
||||
// Apply and slice
|
||||
Tensor src_v = src.compose(layout_vt)(make_coord(_,tid),_);
|
||||
Tensor dst_v = dst.compose(layout_vt)(make_coord(_,tid),_);
|
||||
|
||||
// Should account for vec_bits < 8 and/or vec_elem <= 1
|
||||
// And also account for subbyte types, which could cause race conditions
|
||||
// Want to ENFORCE sufficient vectorization in those cases
|
||||
static_assert((vec_bits >= 8), "No support for subbyte copying");
|
||||
using VecType = uint_bit_t<vec_bits>;
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print(" "); print("NumThreads: "); print(NumThreads); print("\n");
|
||||
print(" "); print("src: "); print(src); print("\n");
|
||||
print(" "); print("dst: "); print(dst); print("\n");
|
||||
print(" "); print("common_layout: "); print(common_layout); print("\n");
|
||||
print(" "); print("full_perm: "); print(full_perm); print("\n");
|
||||
print(" "); print("Used vector: "); print(vec_elem); print("\n");
|
||||
print(" "); print("Used threads: "); print(vec_thrs); print("\n");
|
||||
print(" "); print("layout_vt: "); print(layout_vt); print("\n");
|
||||
print(" "); print("src.compose(layout_vt): "); print(src.compose(layout_vt)); print("\n");
|
||||
print(" "); print("dst.compose(layout_vt): "); print(dst.compose(layout_vt)); print("\n");
|
||||
print(" "); print("src_v: "); print(src_v); print("\n");
|
||||
print(" "); print("dst_v: "); print(dst_v); print("\n");
|
||||
print(" "); print("recast<VecType const>(src_v): "); print(recast<VecType const>(src_v)); print("\n");
|
||||
print(" "); print("recast<VecType const>(dst_v): "); print(recast<VecType const>(dst_v)); print("\n");
|
||||
}
|
||||
#ifdef __CUDA_ARCH__
|
||||
__syncthreads();
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// If we're using all threads (static) or the tid is in in-range (dynamic)
|
||||
if (vec_thrs >= NumThreads or tid < vec_thrs) {
|
||||
return copy_if(TrivialPredTensor{}, recast<VecType const>(src_v), recast<VecType>(dst_v));
|
||||
}
|
||||
}
|
||||
|
||||
template <uint32_t NumThreads,
|
||||
class SrcEngine, class SrcLayout,
|
||||
class DstEngine, class DstLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_copy(uint32_t const& tid,
|
||||
Tensor<SrcEngine, SrcLayout> const& src,
|
||||
Tensor<DstEngine, DstLayout> & dst)
|
||||
{
|
||||
constexpr uint32_t MaxVecBits = sizeof_bits_v<typename SrcEngine::value_type>;
|
||||
return cooperative_copy<NumThreads, MaxVecBits>(tid, src, dst);
|
||||
}
|
||||
|
||||
// Accept mutable temporaries
|
||||
template <uint32_t NumThreads,
|
||||
class SrcEngine, class SrcLayout,
|
||||
class DstEngine, class DstLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_copy(uint32_t const& tid,
|
||||
Tensor<SrcEngine, SrcLayout> const& src,
|
||||
Tensor<DstEngine, DstLayout> && dst)
|
||||
{
|
||||
return cooperative_copy<NumThreads>(tid, src, dst);
|
||||
}
|
||||
|
||||
// Accept mutable temporaries
|
||||
template <uint32_t NumThreads,
|
||||
uint32_t MaxVecBits,
|
||||
class SrcEngine, class SrcLayout,
|
||||
class DstEngine, class DstLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_copy(uint32_t const& tid,
|
||||
Tensor<SrcEngine, SrcLayout> const& src,
|
||||
Tensor<DstEngine, DstLayout> && dst)
|
||||
{
|
||||
return cooperative_copy<NumThreads, MaxVecBits>(tid, src, dst);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
@@ -0,0 +1,326 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
|
||||
#include <cute/atom/mma_atom.hpp>
|
||||
|
||||
#include <cute/algorithm/functional.hpp>
|
||||
#include <cute/algorithm/gemm.hpp>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
//
|
||||
// Collective Shared-Memory GEMMs
|
||||
//
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
class ALoadTransformOp, class BLoadTransformOp,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC,
|
||||
ALoadTransformOp const& sA_load_op /* transforms A values before used in GEMM */,
|
||||
BLoadTransformOp const& sB_load_op /* transforms B values before used in GEMM */)
|
||||
{
|
||||
CUTE_STATIC_ASSERT_V(size<0>(sA) == size<0>(sC)); // AM == CM
|
||||
CUTE_STATIC_ASSERT_V(size<0>(sB) == size<1>(sC)); // BN == CN
|
||||
CUTE_STATIC_ASSERT_V(size<1>(sA) == size<1>(sB)); // AK == BK
|
||||
|
||||
using TypeA = typename TA::value_type;
|
||||
using TypeB = typename TB::value_type;
|
||||
using TypeC = typename TC::value_type;
|
||||
|
||||
static_assert(is_same_v<decay_t<invoke_result_t<ALoadTransformOp, TypeA>>, TypeA>,
|
||||
"ALoadTransformOp functor must accept and return value of type TA::value_type");
|
||||
static_assert(is_same_v<decay_t<invoke_result_t<BLoadTransformOp, TypeB>>, TypeB>,
|
||||
"BLoadTransformOp functor must accept and return value of type TB::value_type");
|
||||
|
||||
// Original, static size of the problem
|
||||
auto M = size<0>(sC);
|
||||
auto N = size<1>(sC);
|
||||
auto K = size<1>(sA);
|
||||
|
||||
// Block size of the compute tile
|
||||
auto BLK_M = tile_size<0>(thr_mma);
|
||||
auto BLK_N = tile_size<1>(thr_mma);
|
||||
auto BLK_K = tile_size<2>(thr_mma);
|
||||
|
||||
// Compute the "residues"
|
||||
auto m_residue = M - BLK_M * (ceil_div(M, BLK_M) - Int<1>{}); // (0,BLK_M]
|
||||
auto n_residue = N - BLK_N * (ceil_div(N, BLK_N) - Int<1>{}); // (0,BLK_N]
|
||||
auto k_residue = K - BLK_K * (ceil_div(K, BLK_K) ); // (-BLK_K,0]
|
||||
|
||||
// Shift the origin so k_residue is zeroth tile
|
||||
sA.data() = &sA(0,k_residue);
|
||||
sB.data() = &sB(0,k_residue);
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
printf("%d in BLK_M (%d)\n", int(m_residue), int(BLK_M));
|
||||
printf("%d in BLK_N (%d)\n", int(n_residue), int(BLK_N));
|
||||
printf("%d in BLK_K (%d)\n", int(k_residue), int(BLK_K));
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
// MMA Partitioning
|
||||
//
|
||||
|
||||
// Round the layout extents up to BLK_X
|
||||
Tensor rounded_sA = sA.compose(make_shape(ceil_div(M, BLK_M) * BLK_M, ceil_div(K, BLK_K) * BLK_K));
|
||||
Tensor rounded_sB = sB.compose(make_shape(ceil_div(N, BLK_N) * BLK_N, ceil_div(K, BLK_K) * BLK_K));
|
||||
Tensor rounded_sC = sC.compose(make_shape(ceil_div(M, BLK_M) * BLK_M, ceil_div(N, BLK_N) * BLK_N));
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print("rounded_sA: "); print(rounded_sA); print("\n");
|
||||
print("rounded_sB: "); print(rounded_sB); print("\n");
|
||||
print("rounded_sC: "); print(rounded_sC); print("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
// Partition the sA and sB tiles across the threads for the MMA
|
||||
Tensor tCsA = thr_mma.partition_A(rounded_sA); // (MMA,MMA_M,MMA_K)
|
||||
Tensor tCsB = thr_mma.partition_B(rounded_sB); // (MMA,MMA_N,MMA_K)
|
||||
Tensor tCsC = thr_mma.partition_C(rounded_sC); // (MMA,MMA_M,MMA_N)
|
||||
// Create register tensors for the MMA to operate on
|
||||
Tensor tCrA = thr_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K)
|
||||
Tensor tCrB = thr_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K)
|
||||
Tensor tCrC = thr_mma.make_fragment_C(tCsC); // (MMA,MMA_M,MMA_N)
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print("tCsA: "); print(tCsA); print("\n");
|
||||
print("tCsB: "); print(tCsB); print("\n");
|
||||
print("tCsC: "); print(tCsC); print("\n");
|
||||
print("tCrA: "); print(tCrA); print("\n");
|
||||
print("tCrB: "); print(tCrB); print("\n");
|
||||
print("tCrC: "); print(tCrC); print("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
// PREDICATION
|
||||
//
|
||||
|
||||
// Allocate the preds for only the MMA-mode of tCsA and tCsB
|
||||
Tensor tCpA = make_tensor<bool>(size<0>(tCsA));
|
||||
Tensor tCpB = make_tensor<bool>(size<0>(tCsB));
|
||||
|
||||
// Create coordinate tensors on a single compute block for predication
|
||||
Tensor cA = make_identity_tensor(make_shape(BLK_M, BLK_K)); // (BLK_M,BLK_K) -> (blk_m,blk_k)
|
||||
Tensor cB = make_identity_tensor(make_shape(BLK_N, BLK_K)); // (BLK_M,BLK_K) -> (blk_n,blk_k)
|
||||
|
||||
// Repeat partitioning with thr_mma
|
||||
Tensor tCcA = thr_mma.partition_A(cA); // (MMA,1,1) -> (blk_m,blk_k)
|
||||
Tensor tCcB = thr_mma.partition_B(cB); // (MMA,1,1) -> (blk_n,blk_k)
|
||||
|
||||
// Populate the m and n predicates
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tCpA); ++i) {
|
||||
tCpA(i) = elem_less(get<0>(tCcA(i)), m_residue);
|
||||
}
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tCpB); ++i) {
|
||||
tCpB(i) = elem_less(get<0>(tCcB(i)), n_residue);
|
||||
}
|
||||
|
||||
#if 0
|
||||
printf("Thr %d: A(%d,%d):%d B(%d,%d):%d\n",
|
||||
threadIdx.x,
|
||||
int(get<0>(tCcA(0))), int(get<1>(tCcA(0))), int(tCpA(0)),
|
||||
int(get<0>(tCcB(0))), int(get<1>(tCcB(0))), int(tCpB(0)));
|
||||
#endif
|
||||
|
||||
//
|
||||
// PREFETCH k_block = 0 (with k-predication)
|
||||
//
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsA); ++i) { // Copy MMA_I
|
||||
if (k_residue == 0 || get<1>(tCcA(i)) >= -k_residue) { // k_block = 0, predicated on k
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsA); ++m) { // Copy MMA_M, predicated on m
|
||||
tCrA(i,m,0) = (m_residue == BLK_M || m < size<1>(tCsA)-1 || tCpA(i)) ? sA_load_op(tCsA(i,m,0)) : TypeA{};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsB); ++i) { // Copy MMA_I
|
||||
if (k_residue == 0 || get<1>(tCcB(i)) >= -k_residue) { // k_block = 0, predicated on k
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<1>(tCsB); ++n) { // Copy MMA_N, predicated on n
|
||||
tCrB(i,n,0) = (n_residue == BLK_N || n < size<1>(tCsB)-1 || tCpB(i)) ? sB_load_op(tCsB(i,n,0)) : TypeB{};
|
||||
}
|
||||
}
|
||||
}
|
||||
//
|
||||
// MAINLOOP
|
||||
//
|
||||
|
||||
// Clear accumulators
|
||||
clear(tCrC);
|
||||
|
||||
constexpr int K_BLOCK_MAX = size<2>(tCrA);
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block)
|
||||
{
|
||||
// static-if load the next k_block. No k-predication required on these loads.
|
||||
if (k_block < K_BLOCK_MAX-1)
|
||||
{
|
||||
// Load the next k_block
|
||||
int k_next = k_block + 1;
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsA); ++m) { // Copy MMA_M
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsA); ++i) { // Copy_if MMA_I predicated on m
|
||||
tCrA(i,m,k_next) = (m_residue == BLK_M || m < size<1>(tCsA)-1 || tCpA(i)) ? sA_load_op(tCsA(i,m,k_next)) : TypeA{};
|
||||
}
|
||||
}
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<1>(tCsB); ++n) { // Copy MMA_N
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsB); ++i) { // Copy MMA_I predicated on n
|
||||
tCrB(i,n,k_next) = (n_residue == BLK_N || n < size<1>(tCsB)-1 || tCpB(i)) ? sB_load_op(tCsB(i,n,k_next)) : TypeB{};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GEMM on k_block in registers
|
||||
gemm(thr_mma, tCrA(_,_,k_block), tCrB(_,_,k_block), tCrC);
|
||||
}
|
||||
|
||||
//
|
||||
// Epilogue
|
||||
//
|
||||
|
||||
Tensor cC = make_identity_tensor(make_shape(BLK_M, BLK_N)); // (BLK_M,BLK_N) -> (blk_m,blk_n)
|
||||
Tensor tCcC = thr_mma.partition_C(cC); // (MMA, 1, 1) -> (blk_m,blk_n)
|
||||
|
||||
const bool isBetaZero = (beta == Beta{});
|
||||
|
||||
// Custom axpby_if for now
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsC); ++m)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<2>(tCsC); ++n)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsC); ++i)
|
||||
{
|
||||
if ((m_residue == BLK_M || m < size<1>(tCrC)-1 || get<0>(tCcC(i)) < m_residue) &&
|
||||
(n_residue == BLK_N || n < size<2>(tCrC)-1 || get<1>(tCcC(i)) < n_residue))
|
||||
{
|
||||
tCsC(i,m,n) = isBetaZero ? alpha * static_cast<TypeC>(tCrC(i,m,n)) : alpha * static_cast<TypeC>(tCrC(i,m,n)) + beta * static_cast<TypeC>(tCsC(i,m,n));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC)
|
||||
{
|
||||
cooperative_gemm(thr_mma, alpha, sA, sB, beta, sC, identity() /* sA_load_op */, identity() /* sB_load_op */);
|
||||
}
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
class ALoadTransformOp, class BLoadTransformOp,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC,
|
||||
ALoadTransformOp const& sA_load_op /* transforms A values before used in GEMM */,
|
||||
BLoadTransformOp const& sB_load_op /* transforms B values before used in GEMM */)
|
||||
{
|
||||
cooperative_gemm(thr_mma, alpha, sA, sB, beta, sC, sA_load_op, sB_load_op);
|
||||
}
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC)
|
||||
{
|
||||
cooperative_gemm(thr_mma, alpha, sA, sB, beta, sC, identity() /* sA_load_op */, identity() /* sB_load_op */);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
@@ -145,10 +145,10 @@ copy_if(PrdTensor const& pred,
|
||||
namespace detail {
|
||||
|
||||
// Trait that detects if atom's traits has a member function with(bool)
|
||||
template<typename, typename Enable = void>
|
||||
template <class, class Enable = void>
|
||||
constexpr bool has_with_bool = false;
|
||||
|
||||
template<typename T>
|
||||
template <class T>
|
||||
constexpr bool has_with_bool<T, cute::void_t<decltype(declval<typename T::Traits>().with(declval<bool>()))>> = true;
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cute/numeric/complex.hpp>
|
||||
|
||||
/** C++14 <functional> extensions */
|
||||
|
||||
@@ -46,7 +47,7 @@ struct identity {
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto) operator()(T&& arg) const {
|
||||
return std::forward<T>(arg);
|
||||
return static_cast<T&&>(arg);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -69,7 +70,7 @@ struct constant_fn {
|
||||
template <class T> \
|
||||
CUTE_HOST_DEVICE constexpr \
|
||||
decltype(auto) operator()(T&& arg) const { \
|
||||
return OP std::forward<T>(arg); \
|
||||
return OP static_cast<T&&>(arg); \
|
||||
} \
|
||||
}
|
||||
#define CUTE_RIGHT_UNARY_OP(NAME,OP) \
|
||||
@@ -77,7 +78,7 @@ struct constant_fn {
|
||||
template <class T> \
|
||||
CUTE_HOST_DEVICE constexpr \
|
||||
decltype(auto) operator()(T&& arg) const { \
|
||||
return std::forward<T>(arg) OP ; \
|
||||
return static_cast<T&&>(arg) OP ; \
|
||||
} \
|
||||
}
|
||||
#define CUTE_NAMED_UNARY_OP(NAME,OP) \
|
||||
@@ -85,7 +86,7 @@ struct constant_fn {
|
||||
template <class T> \
|
||||
CUTE_HOST_DEVICE constexpr \
|
||||
decltype(auto) operator()(T&& arg) const { \
|
||||
return OP (std::forward<T>(arg)); \
|
||||
return OP (static_cast<T&&>(arg)); \
|
||||
} \
|
||||
}
|
||||
|
||||
@@ -115,7 +116,7 @@ struct shift_right_const {
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto) operator()(T&& arg) const {
|
||||
return std::forward<T>(arg) >> Shift;
|
||||
return static_cast<T&&>(arg) >> Shift;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -126,7 +127,7 @@ struct shift_left_const {
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto) operator()(T&& arg) const {
|
||||
return std::forward<T>(arg) << Shift;
|
||||
return static_cast<T&&>(arg) << Shift;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -139,7 +140,7 @@ struct shift_left_const {
|
||||
template <class T, class U> \
|
||||
CUTE_HOST_DEVICE constexpr \
|
||||
decltype(auto) operator()(T&& lhs, U&& rhs) const { \
|
||||
return std::forward<T>(lhs) OP std::forward<U>(rhs); \
|
||||
return static_cast<T&&>(lhs) OP static_cast<U&&>(rhs); \
|
||||
} \
|
||||
}
|
||||
#define CUTE_NAMED_BINARY_OP(NAME,OP) \
|
||||
@@ -147,7 +148,7 @@ struct shift_left_const {
|
||||
template <class T, class U> \
|
||||
CUTE_HOST_DEVICE constexpr \
|
||||
decltype(auto) operator()(T&& lhs, U&& rhs) const { \
|
||||
return OP (std::forward<T>(lhs), std::forward<U>(rhs)); \
|
||||
return OP (static_cast<T&&>(lhs), static_cast<U&&>(rhs)); \
|
||||
} \
|
||||
}
|
||||
|
||||
@@ -273,7 +274,7 @@ struct bound_fn {
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto)
|
||||
operator()(T&& arg) {
|
||||
return fn_(arg_, std::forward<T>(arg));
|
||||
return fn_(arg_, static_cast<T&&>(arg));
|
||||
}
|
||||
|
||||
Fn fn_;
|
||||
|
||||
@@ -252,7 +252,7 @@ gemm(MMA_Atom<MMA> const& mma,
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutC_TV{}) == Int<1>{});
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutA_TV{}) == Int<1>{});
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutB_TV{}) == Int<1>{});
|
||||
|
||||
|
||||
gemm(mma,
|
||||
make_tensor(D.data(), prepend<3>(D.layout())), // (1,M,N)
|
||||
make_tensor(A.data(), prepend<3>(A.layout())), // (1,M,K)
|
||||
@@ -451,6 +451,7 @@ gemm(MMA_Atom<MMA> const& mma,
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutC_TV{}) == Int<1>{});
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutA_TV{}) == Int<1>{});
|
||||
CUTE_STATIC_ASSERT_V(size<1>(typename MMA_Atom<MMA>::LayoutB_TV{}) == Int<1>{});
|
||||
|
||||
gemm(mma,
|
||||
make_tensor(D.data(), prepend<3>(D.layout())), // (1,M,N)
|
||||
make_tensor(A.data(), prepend<3>(A.layout())), // (1,M,K)
|
||||
@@ -496,245 +497,4 @@ gemm(MMA_Atom<MMA> const& mma,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Collective Shared-Memory GEMMs
|
||||
//
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
class ALoadTransformOp, class BLoadTransformOp,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC,
|
||||
ALoadTransformOp const& sA_load_op /* transforms A values before used in GEMM */,
|
||||
BLoadTransformOp const& sB_load_op /* transforms B values before used in GEMM */)
|
||||
{
|
||||
CUTE_STATIC_ASSERT_V(size<0>(sA) == size<0>(sC)); // AM == CM
|
||||
CUTE_STATIC_ASSERT_V(size<0>(sB) == size<1>(sC)); // BN == CN
|
||||
CUTE_STATIC_ASSERT_V(size<1>(sA) == size<1>(sB)); // AK == BK
|
||||
|
||||
using TypeA = typename TA::value_type;
|
||||
using TypeB = typename TB::value_type;
|
||||
using TypeC = typename TC::value_type;
|
||||
|
||||
static_assert(is_same_v<decay_t<invoke_result_t<ALoadTransformOp, TypeA>>, TypeA>,
|
||||
"ALoadTransformOp functor must accept and return value of type TA::value_type");
|
||||
static_assert(is_same_v<decay_t<invoke_result_t<BLoadTransformOp, TypeB>>, TypeB>,
|
||||
"BLoadTransformOp functor must accept and return value of type TB::value_type");
|
||||
|
||||
// Original, static size of the problem
|
||||
auto M = size<0>(sC);
|
||||
auto N = size<1>(sC);
|
||||
auto K = size<1>(sA);
|
||||
|
||||
// Block size of the compute tile
|
||||
auto BLK_M = tile_size<0>(thr_mma);
|
||||
auto BLK_N = tile_size<1>(thr_mma);
|
||||
auto BLK_K = tile_size<2>(thr_mma);
|
||||
|
||||
// Compute the "residues"
|
||||
auto m_residue = M - BLK_M * (ceil_div(M, BLK_M) - Int<1>{}); // (0,BLK_M]
|
||||
auto n_residue = N - BLK_N * (ceil_div(N, BLK_N) - Int<1>{}); // (0,BLK_N]
|
||||
auto k_residue = K - BLK_K * (ceil_div(K, BLK_K) ); // (-BLK_K,0]
|
||||
|
||||
// Shift the origin so k_residue is zeroth tile
|
||||
sA.data() = &sA(0,k_residue);
|
||||
sB.data() = &sB(0,k_residue);
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
printf("%d in BLK_M (%d)\n", int(m_residue), int(BLK_M));
|
||||
printf("%d in BLK_N (%d)\n", int(n_residue), int(BLK_N));
|
||||
printf("%d in BLK_K (%d)\n", int(k_residue), int(BLK_K));
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
// MMA Partitioning
|
||||
//
|
||||
|
||||
// Round the layout extents up to BLK_X
|
||||
Tensor rounded_sA = sA.compose(make_shape(ceil_div(M, BLK_M) * BLK_M, ceil_div(K, BLK_K) * BLK_K));
|
||||
Tensor rounded_sB = sB.compose(make_shape(ceil_div(N, BLK_N) * BLK_N, ceil_div(K, BLK_K) * BLK_K));
|
||||
Tensor rounded_sC = sC.compose(make_shape(ceil_div(M, BLK_M) * BLK_M, ceil_div(N, BLK_N) * BLK_N));
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print(rounded_sA.layout()); print("\n");
|
||||
print(rounded_sB.layout()); print("\n");
|
||||
print(rounded_sC.layout()); print("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
// Partition the sA and sB tiles across the threads for the MMA
|
||||
Tensor tCsA = thr_mma.partition_A(rounded_sA); // (MMA,MMA_M,MMA_K)
|
||||
Tensor tCsB = thr_mma.partition_B(rounded_sB); // (MMA,MMA_N,MMA_K)
|
||||
Tensor tCsC = thr_mma.partition_C(rounded_sC); // (MMA,MMA_M,MMA_N)
|
||||
// Create register tensors for the MMA to operate on
|
||||
Tensor tCrA = thr_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K)
|
||||
Tensor tCrB = thr_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K)
|
||||
Tensor tCrC = thr_mma.make_fragment_C(tCsC); // (MMA,MMA_M,MMA_N)
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print(tCsA.layout()); print("\n");
|
||||
print(tCsB.layout()); print("\n");
|
||||
print(tCsC.layout()); print("\n");
|
||||
print(tCrA.layout()); print("\n");
|
||||
print(tCrB.layout()); print("\n");
|
||||
print(tCrC.layout()); print("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
// PREDICATION
|
||||
//
|
||||
|
||||
// Allocate the preds for only the MMA-mode of tCsA and tCsB
|
||||
Tensor tCpA = make_tensor<bool>(size<0>(tCsA));
|
||||
Tensor tCpB = make_tensor<bool>(size<0>(tCsB));
|
||||
|
||||
// Create coordinate tensors on a single compute block for predication
|
||||
Tensor cA = make_identity_tensor(make_shape(BLK_M, BLK_K)); // (BLK_M,BLK_K) -> (blk_m,blk_k)
|
||||
Tensor cB = make_identity_tensor(make_shape(BLK_N, BLK_K)); // (BLK_M,BLK_K) -> (blk_n,blk_k)
|
||||
|
||||
// Repeat partitioning with thr_mma
|
||||
Tensor tCcA = thr_mma.partition_A(cA); // (MMA,1,1) -> (blk_m,blk_k)
|
||||
Tensor tCcB = thr_mma.partition_B(cB); // (MMA,1,1) -> (blk_n,blk_k)
|
||||
|
||||
// Populate the m and n predicates
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tCpA); ++i) {
|
||||
tCpA(i) = elem_less(get<0>(tCcA(i)), m_residue);
|
||||
}
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tCpB); ++i) {
|
||||
tCpB(i) = elem_less(get<0>(tCcB(i)), n_residue);
|
||||
}
|
||||
|
||||
#if 0
|
||||
printf("Thr %d: A(%d,%d):%d B(%d,%d):%d\n",
|
||||
threadIdx.x,
|
||||
int(get<0>(tCcA(0))), int(get<1>(tCcA(0))), int(tCpA(0)),
|
||||
int(get<0>(tCcB(0))), int(get<1>(tCcB(0))), int(tCpB(0)));
|
||||
#endif
|
||||
|
||||
//
|
||||
// PREFETCH k_block = 0 (with k-predication)
|
||||
//
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsA); ++i) { // Copy MMA_I
|
||||
if (k_residue == 0 || get<1>(tCcA(i)) >= -k_residue) { // k_block = 0, predicated on k
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsA); ++m) { // Copy MMA_M, predicated on m
|
||||
tCrA(i,m,0) = (m_residue == BLK_M || m < size<1>(tCsA)-1 || tCpA(i)) ? sA_load_op(tCsA(i,m,0)) : TypeA{};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsB); ++i) { // Copy MMA_I
|
||||
if (k_residue == 0 || get<1>(tCcB(i)) >= -k_residue) { // k_block = 0, predicated on k
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<1>(tCsB); ++n) { // Copy MMA_N, predicated on n
|
||||
tCrB(i,n,0) = (n_residue == BLK_N || n < size<1>(tCsB)-1 || tCpB(i)) ? sB_load_op(tCsB(i,n,0)) : TypeB{};
|
||||
}
|
||||
}
|
||||
}
|
||||
//
|
||||
// MAINLOOP
|
||||
//
|
||||
|
||||
// Clear accumulators
|
||||
clear(tCrC);
|
||||
|
||||
constexpr int K_BLOCK_MAX = size<2>(tCrA);
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block)
|
||||
{
|
||||
// static-if load the next k_block. No k-predication required on these loads.
|
||||
if (k_block < K_BLOCK_MAX-1)
|
||||
{
|
||||
// Load the next k_block
|
||||
int k_next = k_block + 1;
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsA); ++m) { // Copy MMA_M
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsA); ++i) { // Copy_if MMA_I predicated on m
|
||||
tCrA(i,m,k_next) = (m_residue == BLK_M || m < size<1>(tCsA)-1 || tCpA(i)) ? sA_load_op(tCsA(i,m,k_next)) : TypeA{};
|
||||
}
|
||||
}
|
||||
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<1>(tCsB); ++n) { // Copy MMA_N
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsB); ++i) { // Copy MMA_I predicated on n
|
||||
tCrB(i,n,k_next) = (n_residue == BLK_N || n < size<1>(tCsB)-1 || tCpB(i)) ? sB_load_op(tCsB(i,n,k_next)) : TypeB{};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GEMM on k_block in registers
|
||||
gemm(thr_mma, tCrA(_,_,k_block), tCrB(_,_,k_block), tCrC);
|
||||
}
|
||||
|
||||
//
|
||||
// Epilogue
|
||||
//
|
||||
|
||||
Tensor cC = make_identity_tensor(make_shape(BLK_M, BLK_N)); // (BLK_M,BLK_N) -> (blk_m,blk_n)
|
||||
Tensor tCcC = thr_mma.partition_C(cC); // (MMA, 1, 1) -> (blk_m,blk_n)
|
||||
|
||||
const bool isBetaZero = (beta == Beta{});
|
||||
|
||||
// Custom axpby_if for now
|
||||
CUTE_UNROLL
|
||||
for (int m = 0; m < size<1>(tCsC); ++m)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int n = 0; n < size<2>(tCsC); ++n)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size<0>(tCsC); ++i)
|
||||
{
|
||||
if ((m_residue == BLK_M || m < size<1>(tCrC)-1 || get<0>(tCcC(i)) < m_residue) &&
|
||||
(n_residue == BLK_N || n < size<2>(tCrC)-1 || get<1>(tCcC(i)) < n_residue))
|
||||
{
|
||||
tCsC(i,m,n) = isBetaZero ? alpha * tCrC(i,m,n) : alpha * tCrC(i,m,n) + beta * tCsC(i,m,n);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class... Args,
|
||||
class Alpha, class TA, class ALayout, class TB, class BLayout,
|
||||
class Beta, class TC, class CLayout,
|
||||
__CUTE_REQUIRES(ALayout::rank == 2 && is_smem<TA>::value &&
|
||||
BLayout::rank == 2 && is_smem<TB>::value &&
|
||||
CLayout::rank == 2 && is_smem<TC>::value)>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
gemm(ThrMMA<Args...> const& thr_mma,
|
||||
Alpha const& alpha,
|
||||
Tensor<TA, ALayout> sA,
|
||||
Tensor<TB, BLayout> sB,
|
||||
Beta const& beta,
|
||||
Tensor<TC, CLayout> sC)
|
||||
{
|
||||
gemm(thr_mma, alpha, sA, sB, beta, sC, identity() /* sA_load_op */, identity() /* sB_load_op */);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
#include <cute/atom/copy_atom.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
//
|
||||
// Prefetch global tensors into L2
|
||||
//
|
||||
|
||||
template <uint32_t NumThreads, uint32_t FetchBytes = 64,
|
||||
class GEngine, class GLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
cooperative_prefetch(uint32_t const& tid,
|
||||
Tensor<GEngine, GLayout> const& src)
|
||||
{
|
||||
static_assert(is_gmem<GEngine>::value, "Expected global tensor for prefetch");
|
||||
|
||||
constexpr int V = decltype(max_common_vector(src, src))::value;
|
||||
|
||||
if constexpr (V > 1) {
|
||||
// L2 sector is 32B, default fetch granularity is 64B
|
||||
using VecType = conditional_t<(V * sizeof_bits_v<typename GEngine::value_type>) < (FetchBytes * 8),
|
||||
ArrayEngine<typename GEngine::value_type, V>,
|
||||
uint8_t[FetchBytes] >;
|
||||
|
||||
Tensor src_v = recast<VecType const>(src);
|
||||
CUTE_UNROLL
|
||||
for (int i = tid; i < size(src_v); i += NumThreads) {
|
||||
prefetch(raw_pointer_cast(&src_v(i)));
|
||||
}
|
||||
} else {
|
||||
CUTE_UNROLL
|
||||
for (int i = tid; i < size(src); i += NumThreads) {
|
||||
prefetch(raw_pointer_cast(&src(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class GEngine, class GLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
prefetch(Tensor<GEngine, GLayout> const& src)
|
||||
{
|
||||
return cooperative_prefetch<1>(0, src);
|
||||
}
|
||||
|
||||
// Prefetch with copy atom
|
||||
namespace detail {
|
||||
|
||||
template <class CopyOp, class = void>
|
||||
constexpr bool has_prefetch = false;
|
||||
|
||||
template <class CopyOp>
|
||||
constexpr bool has_prefetch<CopyOp, void_t<typename CopyOp::PREFETCH>> = true;
|
||||
|
||||
template <class CopyOp, class = void>
|
||||
constexpr bool is_prefetch = false;
|
||||
|
||||
template <class CopyOp>
|
||||
constexpr bool is_prefetch<CopyOp, void_t<typename CopyOp::PREFETCH>> = is_same_v<CopyOp, typename CopyOp::PREFETCH>;
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
template <class CopyOp, class... CT_Args, class... CA_Args,
|
||||
class GEngine, class GLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
prefetch(Copy_Atom<Copy_Traits<CopyOp, CT_Args...>, CA_Args...> const& atom,
|
||||
Tensor<GEngine, GLayout> const& src)
|
||||
{
|
||||
if constexpr (detail::has_prefetch<CopyOp>) {
|
||||
using Prefetch_Traits = Copy_Traits<typename CopyOp::PREFETCH, CT_Args...>;
|
||||
using Prefetch_Atom = Copy_Atom<Prefetch_Traits, CA_Args...>;
|
||||
Prefetch_Atom prefetch_atom{atom};
|
||||
auto& dst = const_cast<Tensor<GEngine, GLayout>&>(src); // dst is ignored for prefetch atoms
|
||||
return copy(prefetch_atom, src, dst);
|
||||
} else {
|
||||
return prefetch(src);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(CUTE_COPY_ATOM_TMA_SM90_ENABLED)
|
||||
template <class... CT_Args,
|
||||
class SrcEngine, class SrcLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
prefetch(Copy_Traits<SM90_BULK_COPY_AUTO, CT_Args...> const& atom,
|
||||
Tensor<SrcEngine, SrcLayout> const& src)
|
||||
{
|
||||
using SrcType = typename SrcEngine::value_type;
|
||||
static_assert(is_gmem<SrcEngine>::value, "Expected global tensor for L2 prefetch");
|
||||
|
||||
auto tiler = max_common_layout(src, src);
|
||||
constexpr int vec_elem = decltype(size(tiler))::value;
|
||||
constexpr int vec_bits = vec_elem * sizeof_bits_v<SrcType>;
|
||||
static_assert(vec_bits >= 128, "Expected at least 128-bits for BLKCP");
|
||||
|
||||
// Construct a new concrete Atom of the vector size
|
||||
auto bulk_atom = Copy_Atom<Copy_Traits<SM90_BULK_COPY_G2S, Int<vec_bits>>, SrcType>{};
|
||||
|
||||
return prefetch(bulk_atom, logical_divide(src, tiler));
|
||||
}
|
||||
|
||||
// Backwards-compat. Throw out any extra Copy_Atom args.
|
||||
template <class... CT_Args, class... CA_Args,
|
||||
class SrcEngine, class SrcLayout>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
prefetch(Copy_Atom<Copy_Traits<SM90_BULK_COPY_AUTO, CT_Args...>, CA_Args...> const& atom,
|
||||
Tensor<SrcEngine, SrcLayout> const& src)
|
||||
{
|
||||
return prefetch(static_cast<Copy_Traits<SM90_BULK_COPY_AUTO, CT_Args...> const&>(atom), src);
|
||||
}
|
||||
#endif // #if defined(CUTE_COPY_ATOM_TMA_SM90_ENABLED)
|
||||
|
||||
} // end namespace cute
|
||||
@@ -50,7 +50,7 @@ for_each(Tensor<Engine,Layout> const& tensor, UnaryOp&& op)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tensor); ++i) {
|
||||
static_cast<UnaryOp&&>(op)(tensor(i));
|
||||
op(tensor(i));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ for_each(Tensor<Engine,Layout>& tensor, UnaryOp&& op)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tensor); ++i) {
|
||||
static_cast<UnaryOp&&>(op)(tensor(i));
|
||||
op(tensor(i));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
for_each(Tensor<Engine,Layout>&& tensor, UnaryOp&& op)
|
||||
{
|
||||
return for_each(tensor, static_cast<UnaryOp&&>(op));
|
||||
return for_each(tensor, op);
|
||||
}
|
||||
|
||||
//
|
||||
@@ -86,7 +86,7 @@ transform(Tensor<Engine,Layout>& tensor, UnaryOp&& op)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tensor); ++i) {
|
||||
tensor(i) = static_cast<UnaryOp&&>(op)(tensor(i));
|
||||
tensor(i) = op(tensor(i));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,27 +96,34 @@ CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
transform(Tensor<Engine,Layout>&& tensor, UnaryOp&& op)
|
||||
{
|
||||
return transform(tensor, std::forward<UnaryOp>(op));
|
||||
return transform(tensor, op);
|
||||
}
|
||||
|
||||
// Similar to std::transform transforms one tensors and assigns it to another
|
||||
template <class EngineIn, class LayoutIn, class EngineOut, class LayoutOut, class UnaryOp>
|
||||
template <class EngineIn, class LayoutIn,
|
||||
class EngineOut, class LayoutOut,
|
||||
class UnaryOp>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
transform(Tensor<EngineIn,LayoutIn>& tensor_in, Tensor<EngineOut,LayoutOut>& tensor_out, UnaryOp&& op)
|
||||
transform(Tensor<EngineIn, LayoutIn > const& tensor_in,
|
||||
Tensor<EngineOut,LayoutOut> & tensor_out,
|
||||
UnaryOp&& op)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tensor_in); ++i) {
|
||||
tensor_out(i) = static_cast<UnaryOp&&>(op)(tensor_in(i));
|
||||
tensor_out(i) = op(tensor_in(i));
|
||||
}
|
||||
}
|
||||
|
||||
// Accept mutable temporaries
|
||||
template <class EngineIn, class LayoutIn,
|
||||
class EngineOut, class LayoutOut, class UnaryOp>
|
||||
class EngineOut, class LayoutOut,
|
||||
class UnaryOp>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
transform(Tensor<EngineIn,LayoutIn>&& tensor_in, Tensor<EngineOut,LayoutOut>&& tensor_out, UnaryOp&& op)
|
||||
transform(Tensor<EngineIn, LayoutIn > const& tensor_in,
|
||||
Tensor<EngineOut,LayoutOut> && tensor_out,
|
||||
UnaryOp&& op)
|
||||
{
|
||||
return transform(tensor_in, tensor_out, op);
|
||||
}
|
||||
@@ -127,29 +134,31 @@ transform(Tensor<EngineIn,LayoutIn>&& tensor_in, Tensor<EngineOut,LayoutOut>&& t
|
||||
// assigns it to tensor_out
|
||||
template <class EngineIn1, class LayoutIn1,
|
||||
class EngineIn2, class LayoutIn2,
|
||||
class EngineOut, class LayoutOut, class BinaryOp>
|
||||
class EngineOut, class LayoutOut,
|
||||
class BinaryOp>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
transform(Tensor<EngineIn1,LayoutIn1>& tensor_in1,
|
||||
Tensor<EngineIn2,LayoutIn2>& tensor_in2,
|
||||
Tensor<EngineOut,LayoutOut>& tensor_out,
|
||||
transform(Tensor<EngineIn1,LayoutIn1> const& tensor_in1,
|
||||
Tensor<EngineIn2,LayoutIn2> const& tensor_in2,
|
||||
Tensor<EngineOut,LayoutOut> & tensor_out,
|
||||
BinaryOp&& op)
|
||||
{
|
||||
CUTE_UNROLL
|
||||
for (int i = 0; i < size(tensor_in1); ++i) {
|
||||
tensor_out(i) = static_cast<BinaryOp&&>(op)(tensor_in1(i), tensor_in2(i));
|
||||
tensor_out(i) = op(tensor_in1(i), tensor_in2(i));
|
||||
}
|
||||
}
|
||||
|
||||
// Accept mutable temporaries
|
||||
template <class EngineIn1, class LayoutIn1,
|
||||
class EngineIn2, class LayoutIn2,
|
||||
class EngineOut, class LayoutOut, class BinaryOp>
|
||||
class EngineOut, class LayoutOut,
|
||||
class BinaryOp>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
transform(Tensor<EngineIn1,LayoutIn1>&& tensor_in1,
|
||||
Tensor<EngineIn2,LayoutIn2>&& tensor_in2,
|
||||
Tensor<EngineOut,LayoutOut>&& tensor_out,
|
||||
transform(Tensor<EngineIn1,LayoutIn1> const& tensor_in1,
|
||||
Tensor<EngineIn2,LayoutIn2> const& tensor_in2,
|
||||
Tensor<EngineOut,LayoutOut> && tensor_out,
|
||||
BinaryOp&& op)
|
||||
{
|
||||
return transform(tensor_in1, tensor_in2, tensor_out, op);
|
||||
|
||||
@@ -204,36 +204,6 @@ for_each_leaf(T&& t, F&& f)
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
//
|
||||
// For Sequence
|
||||
// (s, t, f) => (f(t[s_0]),f(t[s_1]),...,f(t[s_n]))
|
||||
//
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <int... I, class F>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
for_sequence(seq<I...> const&, F&& f) {
|
||||
(f(Int<I>{}), ...);
|
||||
}
|
||||
|
||||
}; // end namespace detail
|
||||
|
||||
template <int... I, class T, class F>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
for_sequence(seq<I...> const& s, T&& t, F&& f) {
|
||||
detail::for_sequence(s, [&](auto&& i){ f(get<remove_cvref_t<decltype(i)>::value>(static_cast<T&&>(t))); });
|
||||
}
|
||||
|
||||
template <int I, class T, class F>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
for_sequence(T&& t, F&& f) {
|
||||
for_sequence(make_seq<I>{}, static_cast<T&&>(t), static_cast<F&&>(f));
|
||||
}
|
||||
|
||||
//
|
||||
// Transform
|
||||
// (t, f) => (f(t_0),f(t_1),...,f(t_n))
|
||||
@@ -551,15 +521,15 @@ take(T const& t)
|
||||
template <int... I, class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
select(T const & t)
|
||||
select(T const& t)
|
||||
{
|
||||
return cute::make_tuple(get<I>(t)...);
|
||||
}
|
||||
|
||||
template <class T, typename Indices>
|
||||
template <class T, class Indices>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
select(T const & t, Indices const & indices)
|
||||
select(T const& t, Indices const& indices)
|
||||
{
|
||||
if constexpr (is_tuple<Indices>::value) {
|
||||
return cute::transform(indices, [&t](auto i) { return select(t, i); });
|
||||
@@ -655,7 +625,7 @@ flatten(T const& t)
|
||||
|
||||
namespace detail {
|
||||
|
||||
template<class FlatTuple, class TargetProfile>
|
||||
template <class FlatTuple, class TargetProfile>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
unflatten_impl(FlatTuple const& flat_tuple, TargetProfile const& target_profile)
|
||||
@@ -680,7 +650,7 @@ unflatten_impl(FlatTuple const& flat_tuple, TargetProfile const& target_profile)
|
||||
// @pre rank(flatten(@a target_profile)) == rank(@a flat_tuple)
|
||||
// @post congruent(@a result, @a target_profile)
|
||||
// @post flatten(@a result) == @a flat_tuple
|
||||
template<class FlatTuple, class TargetProfile>
|
||||
template <class FlatTuple, class TargetProfile>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
unflatten(FlatTuple const& flat_tuple, TargetProfile const& target_profile)
|
||||
@@ -865,6 +835,7 @@ append(T const& a, X const& x)
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
template <class T, class X>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -902,6 +873,7 @@ prepend(T const& a, X const& x)
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
template <class T, class X>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -1105,14 +1077,13 @@ zip2_by(T const& t, TG const& guide)
|
||||
|
||||
/// @return A tuple of the elements of @c t in reverse order.
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr auto
|
||||
reverse(T const& t) {
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
reverse(T const& t)
|
||||
{
|
||||
if constexpr (is_tuple<T>::value) {
|
||||
return detail::apply(t, [] (auto const&... a) {
|
||||
return cute::make_tuple(a...);
|
||||
}, tuple_rseq<T>{});
|
||||
}
|
||||
else {
|
||||
return detail::apply(t, [](auto const&... a){ return cute::make_tuple(a...); }, tuple_rseq<T>{});
|
||||
} else {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ CUTE_DEVICE void cluster_arrive_relaxed()
|
||||
#if defined(CUTE_ARCH_CLUSTER_SM90_ENABLED)
|
||||
asm volatile("barrier.cluster.arrive.relaxed.aligned;\n" : : );
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
CUTE_INVALID_CONTROL_PATH("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ CUTE_DEVICE void cluster_arrive()
|
||||
#if defined(CUTE_ARCH_CLUSTER_SM90_ENABLED)
|
||||
asm volatile("barrier.cluster.arrive.aligned;\n" : : );
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
CUTE_INVALID_CONTROL_PATH("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ CUTE_DEVICE void cluster_wait()
|
||||
#if defined(CUTE_ARCH_CLUSTER_SM90_ENABLED)
|
||||
asm volatile("barrier.cluster.wait.aligned;\n" : : );
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
CUTE_INVALID_CONTROL_PATH("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ CUTE_DEVICE void cluster_sync()
|
||||
cluster_arrive();
|
||||
cluster_wait();
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
CUTE_INVALID_CONTROL_PATH("CUTE_ARCH_CLUSTER_SM90_ENABLED is not defined");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -94,7 +94,7 @@ CUTE_DEVICE dim3 cluster_grid_dims()
|
||||
// MSVC requires protecting use of gridDim with __CUDA_ARCH__.
|
||||
return gridDim;
|
||||
#elif defined(_MSC_VER)
|
||||
CUTE_RUNTIME_ASSERT("cluster_grid_dims() can only be called on device");
|
||||
CUTE_INVALID_CONTROL_PATH("cluster_grid_dims() can only be called on device");
|
||||
return {0, 0, 0};
|
||||
#else
|
||||
return {0, 0, 0};
|
||||
@@ -114,7 +114,7 @@ CUTE_DEVICE dim3 cluster_id_in_grid()
|
||||
// MSVC requires protecting use of blockIdx with __CUDA_ARCH__.
|
||||
return blockIdx;
|
||||
#elif defined(_MSC_VER)
|
||||
CUTE_RUNTIME_ASSERT("cluster_id_in_grid() can only be called on device");
|
||||
CUTE_INVALID_CONTROL_PATH("cluster_id_in_grid() can only be called on device");
|
||||
return {0, 0, 0};
|
||||
#else
|
||||
return {0, 0, 0};
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/arch/util.hpp>
|
||||
#include <cute/numeric/int.hpp>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -89,4 +89,17 @@ using AutoVectorizingCopy = AutoVectorizingCopyWithAssumedAlignment<8>;
|
||||
// Alias
|
||||
using DefaultCopy = AutoVectorizingCopy;
|
||||
|
||||
|
||||
//
|
||||
// Global memory prefetch into L2
|
||||
//
|
||||
|
||||
CUTE_HOST_DEVICE static void
|
||||
prefetch(void const* gmem_ptr)
|
||||
{
|
||||
#if defined(__CUDA_ARCH__)
|
||||
asm volatile("prefetch.global.L2 [%0];\n" : : "l"(gmem_ptr) : "memory");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
|
||||
@@ -78,7 +78,7 @@ struct SM75_U32x1_LDSM_N
|
||||
: "=r"(dst)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -98,7 +98,7 @@ struct SM75_U32x2_LDSM_N
|
||||
: "=r"(dst0), "=r"(dst1)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -118,7 +118,7 @@ struct SM75_U32x4_LDSM_N
|
||||
: "=r"(dst0), "=r"(dst1), "=r"(dst2), "=r"(dst3)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -138,7 +138,7 @@ struct SM75_U16x2_LDSM_T
|
||||
: "=r"(dst)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -158,7 +158,7 @@ struct SM75_U16x4_LDSM_T
|
||||
: "=r"(dst0), "=r"(dst1)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -178,7 +178,7 @@ struct SM75_U16x8_LDSM_T
|
||||
: "=r"(dst0), "=r"(dst1), "=r"(dst2), "=r"(dst3)
|
||||
: "r"(smem_int_ptr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use ldmatrix without CUTE_ARCH_LDSM_SM75_ACTIVATED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -64,7 +64,7 @@ struct SM80_CP_ASYNC_CACHEALWAYS
|
||||
"l"(gmem_ptr),
|
||||
"n"(sizeof(TS)));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Support for cp.async instructions has not been enabled");
|
||||
CUTE_INVALID_CONTROL_PATH("Support for cp.async instructions has not been enabled");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -91,7 +91,7 @@ struct SM80_CP_ASYNC_CACHEGLOBAL
|
||||
"l"(gmem_ptr),
|
||||
"n"(sizeof(TS)));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Support for cp.async instructions has not been enabled");
|
||||
CUTE_INVALID_CONTROL_PATH("Support for cp.async instructions has not been enabled");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -121,7 +121,7 @@ struct SM80_CP_ASYNC_CACHEALWAYS_ZFILL
|
||||
"n"(sizeof(TS)),
|
||||
"r"(src_size));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Support for cp.async instructions has not been enabled");
|
||||
CUTE_INVALID_CONTROL_PATH("Support for cp.async instructions has not been enabled");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -151,7 +151,7 @@ struct SM80_CP_ASYNC_CACHEGLOBAL_ZFILL
|
||||
"n"(sizeof(TS)),
|
||||
"r"(src_size));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Support for cp.async instructions has not been enabled");
|
||||
CUTE_INVALID_CONTROL_PATH("Support for cp.async instructions has not been enabled");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -63,7 +63,7 @@ struct SM90_U32x1_STSM_N
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -83,7 +83,7 @@ struct SM90_U32x2_STSM_N
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src0), "r"(src1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -103,7 +103,7 @@ struct SM90_U32x4_STSM_N
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src0), "r"(src1), "r"(src2), "r"(src3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -123,7 +123,7 @@ struct SM90_U16x2_STSM_T
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -143,7 +143,7 @@ struct SM90_U16x4_STSM_T
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src0), "r"(src1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -163,7 +163,7 @@ struct SM90_U16x8_STSM_T
|
||||
:: "r"(smem_int_ptr),
|
||||
"r"(src0), "r"(src1), "r"(src2), "r"(src3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use stmatrix without CUTE_ARCH_STSM_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -43,8 +43,7 @@
|
||||
#include <cute/container/alignment.hpp>
|
||||
#include <cute/container/bit_field.hpp>
|
||||
#include <cute/container/array.hpp>
|
||||
#include <cute/numeric/int.hpp> // to_Format<[u]intX>
|
||||
#include <cute/numeric/half.hpp> // to_Format<half_t>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -177,8 +176,10 @@ to_CUtensorMapSwizzle(SmemSwizzleBits const& t) {
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 12) && !defined(__CUDACC_RTC__)
|
||||
using TmaDescriptor = CUtensorMap;
|
||||
using Im2ColTmaDescriptor = CUtensorMap;
|
||||
#else
|
||||
using TmaDescriptor = struct alignas(64) { char bytes[128]; };
|
||||
using Im2ColTmaDescriptor = struct alignas(64) { char bytes[128]; };
|
||||
#endif
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/// Initiates a TensorMap Prefetch
|
||||
@@ -197,7 +198,7 @@ prefetch_tma_descriptor(TmaDescriptor const* desc_ptr)
|
||||
: "l"(gmem_int_desc)
|
||||
: "memory");
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Trying to use TMA Descriptor Prefetch without CUTE_ARCH_TMA_SM90_ENABLED.");
|
||||
CUTE_INVALID_CONTROL_PATH("Trying to use TMA Descriptor Prefetch without CUTE_ARCH_TMA_SM90_ENABLED.");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -208,7 +209,7 @@ prefetch_tma_descriptor(TmaDescriptor const* desc_ptr)
|
||||
// Replace tensor pointer directly in GMEM
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
tma_descriptor_replace_addr_in_global_mem(TmaDescriptor const* desc_ptr,
|
||||
tma_descriptor_replace_addr_in_global_mem(TmaDescriptor const* desc_ptr,
|
||||
void const* const new_tensor_ptr)
|
||||
{
|
||||
#if defined(CUTE_ARCH_DEVICE_MODIFIABLE_TMA_SM90_ENABLED)
|
||||
@@ -218,14 +219,14 @@ tma_descriptor_replace_addr_in_global_mem(TmaDescriptor const* desc_ptr,
|
||||
"tensormap.replace.tile.global_address.global.b1024.b64 [%0], %1;"
|
||||
:: "l"(gmem_int_desc), "l"(new_desc_addr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Replace tensor pointer by bringing the tensormap from GMEM into the shared memory
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
tma_descriptor_replace_addr_in_shared_mem(TmaDescriptor& smem_desc,
|
||||
tma_descriptor_replace_addr_in_shared_mem(TmaDescriptor& smem_desc,
|
||||
void const* const new_tensor_ptr)
|
||||
{
|
||||
#if defined(CUTE_ARCH_DEVICE_MODIFIABLE_TMA_SM90_ENABLED)
|
||||
@@ -239,7 +240,7 @@ tma_descriptor_replace_addr_in_shared_mem(TmaDescriptor& smem_desc,
|
||||
"tensormap.replace.tile.global_address.shared::cta.b1024.b64 [%0], %1;"
|
||||
:: "l"(smem_int64_desc), "l"(new_desc_addr));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -273,7 +274,7 @@ tma_descriptor_replace_dims_strides_in_shared_mem(TmaDescriptor
|
||||
"tensormap.replace.tile.global_stride.shared::cta.b1024.b64 [%0], 1, %1;"
|
||||
:: "l"(smem_int64_desc), "l"(prob_stride[2] >> 4));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -292,7 +293,7 @@ tma_descriptor_cp_fence_release(TmaDescriptor const* gmem_desc_ptr, TmaDescripto
|
||||
"tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned [%0], [%1], 128;"
|
||||
:: "l"(gmem_int_desc), "r"(smem_int_desc));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -307,7 +308,7 @@ tma_descriptor_fence_release()
|
||||
#if defined(CUTE_ARCH_DEVICE_MODIFIABLE_TMA_SM90_ENABLED)
|
||||
asm volatile ("fence.proxy.tensormap::generic.release.gpu;");
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -332,7 +333,7 @@ tma_descriptor_fence_acquire(TmaDescriptor const* desc_ptr)
|
||||
: "l"(gmem_int_desc), "l"(gmem_int_desc)
|
||||
: "memory");
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
CUTE_INVALID_CONTROL_PATH("Using TMA Descriptor modification without CUTE_ARCH_TMA_SM90_ENABLED and CUDA 12.3");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
+596
-182
File diff suppressed because it is too large
Load Diff
@@ -58,7 +58,7 @@ struct SM61_DP4A
|
||||
: "=r"(d)
|
||||
: "r"(a), "r"(b), "r"(c));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM61_DP4A without CUTE_ARCH_MMA_SM61_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM61_DP4A without CUTE_ARCH_MMA_SM61_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -79,7 +79,7 @@ struct SM61_DP2A
|
||||
: "=r"(d)
|
||||
: "r"(a), "r"(b), "r"(c));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM61_DP2A without CUTE_ARCH_MMA_SM61_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM61_DP2A without CUTE_ARCH_MMA_SM61_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -74,7 +74,7 @@ struct SM70_8x8x4_F16F16F16F16_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F16F16F16F16_TN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F16F16F16F16_TN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -106,7 +106,7 @@ struct SM70_8x8x4_F16F16F16F16_NT
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F16F16F16F16_NT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F16F16F16F16_NT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -138,7 +138,7 @@ struct SM70_8x8x4_F16F16F16F16_NN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F16F16F16F16_NN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F16F16F16F16_NN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -170,7 +170,7 @@ struct SM70_8x8x4_F16F16F16F16_TT
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F16F16F16F16_TT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F16F16F16F16_TT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -210,7 +210,7 @@ struct SM70_8x8x4_F32F16F16F32_TN
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3),
|
||||
"f"(c4), "f"(c5), "f"(c6), "f"(c7));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F32F16F16F32_TN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F32F16F16F32_TN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -246,7 +246,7 @@ struct SM70_8x8x4_F32F16F16F32_NT
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3),
|
||||
"f"(c4), "f"(c5), "f"(c6), "f"(c7));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F32F16F16F32_NT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F32F16F16F32_NT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -282,7 +282,7 @@ struct SM70_8x8x4_F32F16F16F32_NN
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3),
|
||||
"f"(c4), "f"(c5), "f"(c6), "f"(c7));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F32F16F16F32_NN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F32F16F16F32_NN without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -318,7 +318,7 @@ struct SM70_8x8x4_F32F16F16F32_TT
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3),
|
||||
"f"(c4), "f"(c5), "f"(c6), "f"(c7));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM70_8x8x4_F32F16F16F32_TT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM70_8x8x4_F32F16F16F32_TT without CUTE_ARCH_MMA_SM70_ENABLED");
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ struct SM75_16x8x8_F32F16F16F32_TN
|
||||
"r"(b0),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM75_16x8x8_F32F16F16F32_TN without CUTE_ARCH_MMA_SM75_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM75_16x8x8_F32F16F16F32_TN without CUTE_ARCH_MMA_SM75_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -110,7 +110,7 @@ struct SM75_8x8x16_S32S8S8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM75_8x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM75_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM75_8x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM75_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/arch/mma.hpp>
|
||||
#include <cute/numeric/complex.hpp>
|
||||
|
||||
// Config
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800))
|
||||
@@ -80,7 +81,7 @@ struct SM80_16x8x8_F16F16F16F16_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x8_F16F16F16F16_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x8_F16F16F16F16_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -113,7 +114,7 @@ struct SM80_16x8x16_F16F16F16F16_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_F16F16F16F16_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_F16F16F16F16_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -146,7 +147,7 @@ struct SM80_16x8x8_F32F16F16F32_TN
|
||||
"r"(b0),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x8_F32F16F16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x8_F32F16F16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -179,7 +180,7 @@ struct SM80_16x8x16_F32F16F16F32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_F32F16F16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_F32F16F16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -212,7 +213,7 @@ struct SM80_16x8x8_F32BF16BF16F32_TN
|
||||
"r"(b0),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x8_F32BF16BF16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x8_F32BF16BF16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -245,7 +246,7 @@ struct SM80_16x8x16_F32BF16BF16F32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_F32BF16BF16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_F32BF16BF16F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -278,7 +279,7 @@ struct SM80_16x8x4_F32TF32TF32F32_TN
|
||||
"r"(b0),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x4_F32TF32TF32F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x4_F32TF32TF32F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -311,7 +312,7 @@ struct SM80_16x8x8_F32TF32TF32F32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"f"(c0), "f"(c1), "f"(c2), "f"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x8_F32TF32TF32F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x8_F32TF32TF32F32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -344,7 +345,7 @@ struct SM80_8x8x4_F64F64F64F64_TN
|
||||
"d"(b0),
|
||||
"d"(c0), "d"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x4_F64F64F64F64_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x4_F64F64F64F64_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -385,14 +386,14 @@ struct SM80_8x8x4_C64C64C64C64_TN
|
||||
|
||||
// d.real() = -a.imag() * b.imag() + d.real();
|
||||
SM80_8x8x4_F64F64F64F64_TN::fma(
|
||||
rd0, rd1,
|
||||
rd0, rd1,
|
||||
-a0.imag(),
|
||||
b0.imag(),
|
||||
d0.real(), d1.real());
|
||||
|
||||
// d.imag() = a.real() * b.imag() + d.imag();
|
||||
SM80_8x8x4_F64F64F64F64_TN::fma(
|
||||
id0, id1,
|
||||
id0, id1,
|
||||
a0.real(),
|
||||
b0.imag(),
|
||||
d0.imag(), d1.imag());
|
||||
@@ -412,15 +413,15 @@ struct SM80_8x8x4_GC64C64C64GC64_TN
|
||||
{
|
||||
struct GaussComplex {
|
||||
double t0, t1, t2;
|
||||
|
||||
|
||||
CUTE_HOST_DEVICE //constexpr
|
||||
operator complex<double>() const { return complex<double>(t0 - t1, t2 - t0 - t1); }
|
||||
|
||||
|
||||
CUTE_HOST_DEVICE friend //constexpr
|
||||
complex<double> operator*(GaussComplex const& a, complex<double> const& b) { return static_cast<complex<double>>(a) * b; }
|
||||
CUTE_HOST_DEVICE friend //constexpr
|
||||
complex<double> operator*(complex<double> const& a, GaussComplex const& b) { return b * a; }
|
||||
|
||||
|
||||
CUTE_HOST_DEVICE friend //constexpr
|
||||
complex<double> operator+(GaussComplex const& a, complex<double> const& b) { return static_cast<complex<double>>(a) + b; }
|
||||
CUTE_HOST_DEVICE friend //constexpr
|
||||
@@ -481,7 +482,7 @@ struct SM80_8x8x16_S32S8S8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -514,7 +515,7 @@ struct SM80_8x8x16_S32S8S8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -547,7 +548,7 @@ struct SM80_16x8x16_S32S8S8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -580,7 +581,7 @@ struct SM80_16x8x16_S32S8S8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -613,7 +614,7 @@ struct SM80_16x8x32_S32S8S8S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -646,7 +647,7 @@ struct SM80_16x8x32_S32S8S8S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -679,7 +680,7 @@ struct SM80_8x8x16_S32S8U8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -712,7 +713,7 @@ struct SM80_8x8x16_S32S8U8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -745,7 +746,7 @@ struct SM80_16x8x16_S32S8U8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -778,7 +779,7 @@ struct SM80_16x8x16_S32S8U8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -811,7 +812,7 @@ struct SM80_16x8x32_S32S8U8S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -844,7 +845,7 @@ struct SM80_16x8x32_S32S8U8S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -877,7 +878,7 @@ struct SM80_8x8x16_S32U8S8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -910,7 +911,7 @@ struct SM80_8x8x16_S32U8S8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -943,7 +944,7 @@ struct SM80_16x8x16_S32U8S8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -976,7 +977,7 @@ struct SM80_16x8x16_S32U8S8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1009,7 +1010,7 @@ struct SM80_16x8x32_S32U8S8S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U8S8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1042,7 +1043,7 @@ struct SM80_16x8x32_S32U8S8S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U8S8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1075,7 +1076,7 @@ struct SM80_8x8x16_S32U8U8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1108,7 +1109,7 @@ struct SM80_8x8x16_S32U8U8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x16_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x16_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1141,7 +1142,7 @@ struct SM80_16x8x16_S32U8U8S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1174,7 +1175,7 @@ struct SM80_16x8x16_S32U8U8S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x16_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x16_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1207,7 +1208,7 @@ struct SM80_16x8x32_S32U8U8S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U8U8S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1240,7 +1241,7 @@ struct SM80_16x8x32_S32U8U8S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U8U8S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1273,7 +1274,7 @@ struct SM80_8x8x32_S32S4S4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1306,7 +1307,7 @@ struct SM80_8x8x32_S32S4S4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1339,7 +1340,7 @@ struct SM80_16x8x32_S32S4S4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1372,7 +1373,7 @@ struct SM80_16x8x32_S32S4S4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1405,7 +1406,7 @@ struct SM80_16x8x64_S32S4S4S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32S4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1438,7 +1439,7 @@ struct SM80_16x8x64_S32S4S4S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32S4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1471,7 +1472,7 @@ struct SM80_8x8x32_S32S4U4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1504,7 +1505,7 @@ struct SM80_8x8x32_S32S4U4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1537,7 +1538,7 @@ struct SM80_16x8x32_S32S4U4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1570,7 +1571,7 @@ struct SM80_16x8x32_S32S4U4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1603,7 +1604,7 @@ struct SM80_16x8x64_S32S4U4S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32S4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1636,7 +1637,7 @@ struct SM80_16x8x64_S32S4U4S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32S4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1669,7 +1670,7 @@ struct SM80_8x8x32_S32U4S4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1702,7 +1703,7 @@ struct SM80_8x8x32_S32U4S4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1735,7 +1736,7 @@ struct SM80_16x8x32_S32U4S4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1768,7 +1769,7 @@ struct SM80_16x8x32_S32U4S4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1801,7 +1802,7 @@ struct SM80_16x8x64_S32U4S4S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32U4S4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1834,7 +1835,7 @@ struct SM80_16x8x64_S32U4S4S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32U4S4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1867,7 +1868,7 @@ struct SM80_8x8x32_S32U4U4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1900,7 +1901,7 @@ struct SM80_8x8x32_S32U4U4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x32_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x32_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1933,7 +1934,7 @@ struct SM80_16x8x32_S32U4U4S32_TN
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1966,7 +1967,7 @@ struct SM80_16x8x32_S32U4U4S32_TN_SATURATE
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x32_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x32_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -1999,7 +2000,7 @@ struct SM80_16x8x64_S32U4U4S32_TN
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32U4U4S32_TN without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -2032,7 +2033,7 @@ struct SM80_16x8x64_S32U4U4S32_TN_SATURATE
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x64_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x64_S32U4U4S32_TN_SATURATE without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -2067,7 +2068,7 @@ struct SM80_8x8x128_S32U1U1S32_TN_XORPOPC
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_8x8x128_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_8x8x128_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -2100,7 +2101,7 @@ struct SM80_16x8x128_S32U1U1S32_TN_XORPOPC
|
||||
"r"(b0),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x128_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x128_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -2133,7 +2134,7 @@ struct SM80_16x8x256_S32U1U1S32_TN_XORPOPC
|
||||
"r"(b0), "r"(b1),
|
||||
"r"(c0), "r"(c1), "r"(c2), "r"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM80_16x8x256_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM80_16x8x256_S32U1U1S32_TN_XORPOPC without CUTE_ARCH_MMA_SM80_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
+179
-31
@@ -73,7 +73,7 @@ struct SM90_16x8x4_F64F64F64F64_TN
|
||||
"d"(b0),
|
||||
"d"(c0), "d"(c1), "d"(c2), "d"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM90_16x8x4_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM90_16x8x4_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -106,7 +106,7 @@ struct SM90_16x8x8_F64F64F64F64_TN
|
||||
"d"(b0), "d"(b1),
|
||||
"d"(c0), "d"(c1), "d"(c2), "d"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM90_16x8x8_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM90_16x8x8_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -141,7 +141,7 @@ struct SM90_16x8x16_F64F64F64F64_TN
|
||||
"d"(b0), "d"(b1), "d"(b2), "d"(b3),
|
||||
"d"(c0), "d"(c1), "d"(c2), "d"(c3));
|
||||
#else
|
||||
CUTE_RUNTIME_ASSERT("Attempting to use SM90_16x8x16_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
CUTE_INVALID_CONTROL_PATH("Attempting to use SM90_16x8x16_F64F64F64F64_TN without CUTE_ARCH_MMA_SM90_ENABLED");
|
||||
#endif
|
||||
}
|
||||
};
|
||||
@@ -364,37 +364,185 @@ ss_op_selector()
|
||||
|
||||
// FP16 accumulator
|
||||
if constexpr (is_same_v<ElementC, half_t>) {
|
||||
static_assert(is_same_v<ElementA, half_t>, "Element types for AB must be half if ElementC is half.");
|
||||
static_assert(is_same_v<ElementB, half_t>, "Element types for AB must be half if ElementC is half.");
|
||||
static_assert(size<2>(TileShape_MNK{}) % 16 == 0, "Tile_K must be a multiple of 16.");
|
||||
if constexpr (is_same_v<ElementA, half_t> && is_same_v<ElementB, half_t>) {
|
||||
static_assert(size<2>(TileShape_MNK{}) % 16 == 0, "Tile_K must be a multiple of 16.");
|
||||
|
||||
// Dispatch against the Tile N mode size
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
// Dispatch against the Tile N mode size
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else {
|
||||
static_assert(Tile_N % 8 == 0, "Tile_N must be a multiple of 8.");
|
||||
}
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x16_F16F16F16_SS<MajorA, MajorB, Args...>{};
|
||||
}
|
||||
else {
|
||||
|
||||
// FP8
|
||||
// Input A: float_e4m3_t ; Input B: float_e4m3_t
|
||||
else if constexpr (is_same_v<ElementA, float_e4m3_t> && is_same_v<ElementB, float_e4m3_t>) {
|
||||
static_assert(MajorA == GMMA::Major::K, "MajorA must be GMMA::Major::K for this config.");
|
||||
static_assert(MajorB == GMMA::Major::K, "MajorB must be GMMA::Major::K for this config.");
|
||||
static_assert(size<2>(TileShape_MNK{}) % 32 == 0, "Tile_K must be a multiple of 32.");
|
||||
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x32_F16E4M3E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else {
|
||||
static_assert(Tile_N % 8 == 0, "Tile_N must be a multiple of 8.");
|
||||
}
|
||||
}
|
||||
|
||||
// FP8
|
||||
// Input A: float_e4m3_t ; Input B: float_e5m2_t
|
||||
else if constexpr (is_same_v<ElementA, float_e4m3_t> && is_same_v<ElementB, float_e5m2_t>) {
|
||||
static_assert(MajorA == GMMA::Major::K, "MajorA must be GMMA::Major::K for this config.");
|
||||
static_assert(MajorB == GMMA::Major::K, "MajorB must be GMMA::Major::K for this config.");
|
||||
static_assert(size<2>(TileShape_MNK{}) % 32 == 0, "Tile_K must be a multiple of 32.");
|
||||
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x32_F16E4M3E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else {
|
||||
static_assert(Tile_N % 8 == 0, "Tile_N must be a multiple of 8.");
|
||||
}
|
||||
}
|
||||
|
||||
// FP8
|
||||
// Input A: float_e5m2_t ; Input B: float_e5m2_t
|
||||
else if constexpr (is_same_v<ElementA, float_e5m2_t> && is_same_v<ElementB, float_e5m2_t>) {
|
||||
static_assert(MajorA == GMMA::Major::K, "MajorA must be GMMA::Major::K for this config.");
|
||||
static_assert(MajorB == GMMA::Major::K, "MajorB must be GMMA::Major::K for this config.");
|
||||
static_assert(size<2>(TileShape_MNK{}) % 32 == 0, "Tile_K must be a multiple of 32.");
|
||||
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x32_F16E5M2E5M2_SS_TN<Args...>{};
|
||||
}
|
||||
else {
|
||||
static_assert(Tile_N % 8 == 0, "Tile_N must be a multiple of 8.");
|
||||
}
|
||||
}
|
||||
|
||||
// FP8
|
||||
// Input A: float_e5m2_t ; Input B: float_e4m3_t
|
||||
else if constexpr (is_same_v<ElementA, float_e5m2_t> && is_same_v<ElementB, float_e4m3_t>) {
|
||||
static_assert(MajorA == GMMA::Major::K, "MajorA must be GMMA::Major::K for this config.");
|
||||
static_assert(MajorB == GMMA::Major::K, "MajorB must be GMMA::Major::K for this config.");
|
||||
static_assert(size<2>(TileShape_MNK{}) % 32 == 0, "Tile_K must be a multiple of 32.");
|
||||
|
||||
if constexpr (Tile_N % 256 == 0) {
|
||||
return SM90_64x256x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 192 == 0) {
|
||||
return SM90_64x192x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 128 == 0) {
|
||||
return SM90_64x128x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 96 == 0) {
|
||||
return SM90_64x96x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 64 == 0) {
|
||||
return SM90_64x64x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 32 == 0) {
|
||||
return SM90_64x32x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 16 == 0) {
|
||||
return SM90_64x16x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else if constexpr (Tile_N % 8 == 0) {
|
||||
return SM90_64x8x32_F16E5M2E4M3_SS_TN<Args...>{};
|
||||
}
|
||||
else {
|
||||
static_assert(Tile_N % 8 == 0, "Tile_N must be a multiple of 8.");
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
static_assert(sizeof(ElementA) == 0, "No eligible GMMA operator for request configuration.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+323
-323
File diff suppressed because it is too large
Load Diff
@@ -81,7 +81,7 @@ struct Copy_Atom<Copy_Traits<Args...>, CopyInternalType>
|
||||
CUTE_HOST_DEVICE
|
||||
auto
|
||||
with(TraitsArgs&&... args) const {
|
||||
auto traits = Traits::with(std::forward<TraitsArgs>(args)...);
|
||||
auto traits = Traits::with(static_cast<TraitsArgs&&>(args)...);
|
||||
return Copy_Atom<decltype(traits), CopyInternalType>{traits};
|
||||
}
|
||||
|
||||
@@ -351,7 +351,7 @@ struct ThrCopy
|
||||
partition_S(STensor&& stensor) const {
|
||||
//static_assert(sizeof(typename remove_cvref_t<STensor>::value_type) == sizeof(typename TiledCopy::ValType),
|
||||
// "Expected ValType for tiling SrcTensor.");
|
||||
auto thr_tensor = make_tensor(std::forward<STensor>(stensor).data(), TiledCopy::tidfrg_S(stensor.layout()));
|
||||
auto thr_tensor = make_tensor(static_cast<STensor&&>(stensor).data(), TiledCopy::tidfrg_S(stensor.layout()));
|
||||
return thr_tensor(thr_idx_, _, repeat<rank_v<STensor>>(_));
|
||||
}
|
||||
|
||||
@@ -361,7 +361,7 @@ struct ThrCopy
|
||||
partition_D(DTensor&& dtensor) const {
|
||||
//static_assert(sizeof(typename remove_cvref_t<DTensor>::value_type) == sizeof(typename TiledCopy::ValType),
|
||||
// "Expected ValType for tiling DstTensor.");
|
||||
auto thr_tensor = make_tensor(std::forward<DTensor>(dtensor).data(), TiledCopy::tidfrg_D(dtensor.layout()));
|
||||
auto thr_tensor = make_tensor(static_cast<DTensor&&>(dtensor).data(), TiledCopy::tidfrg_D(dtensor.layout()));
|
||||
return thr_tensor(thr_idx_, _, repeat<rank_v<DTensor>>(_));
|
||||
}
|
||||
|
||||
@@ -371,7 +371,7 @@ struct ThrCopy
|
||||
retile_S(STensor&& stensor) {
|
||||
// static_assert(sizeof(typename remove_cvref_t<STensor>::value_type) == sizeof(typename TiledCopy::ValType),
|
||||
// "Expected ValType for tiling SrcTensor.");
|
||||
return make_tensor(std::forward<STensor>(stensor).data(), TiledCopy::retile(stensor.layout()));
|
||||
return make_tensor(static_cast<STensor&&>(stensor).data(), TiledCopy::retile(stensor.layout()));
|
||||
}
|
||||
|
||||
template <class DTensor>
|
||||
@@ -380,7 +380,7 @@ struct ThrCopy
|
||||
retile_D(DTensor&& dtensor) {
|
||||
// static_assert(sizeof(typename remove_cvref_t<DTensor>::value_type) == sizeof(typename TiledCopy::ValType),
|
||||
// "Expected ValType for tiling DstTensor.");
|
||||
return make_tensor(std::forward<DTensor>(dtensor).data(), TiledCopy::retile(dtensor.layout()));
|
||||
return make_tensor(static_cast<DTensor&&>(dtensor).data(), TiledCopy::retile(dtensor.layout()));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -94,17 +94,55 @@ struct Copy_Traits<AutoVectorizingCopyWithAssumedAlignment<MaxVecBits>>
|
||||
|
||||
namespace detail {
|
||||
|
||||
// Utility for exploding pointers, arrays, or tensors into Operation::copy
|
||||
template <class Operation,
|
||||
class PtrS, int... Is,
|
||||
class PtrD, int... Id>
|
||||
class PtrSrc, int... Is,
|
||||
class PtrDst, int... Id>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_explode(PtrS&& s, int_sequence<Is...>,
|
||||
PtrD&& d, int_sequence<Id...>)
|
||||
copy_explode_index(PtrSrc&& s, int_sequence<Is...>,
|
||||
PtrDst&& d, int_sequence<Id...>)
|
||||
{
|
||||
return Operation::copy(s[Is]..., d[Id]...);
|
||||
}
|
||||
|
||||
// Utility for exploding tuples into ::copy
|
||||
template <class Operation,
|
||||
class TupleArg, int... I>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_explode(TupleArg&& t, int_sequence<I...>)
|
||||
{
|
||||
return Operation::copy(get<I>(static_cast<TupleArg&&>(t))...);
|
||||
}
|
||||
|
||||
template <class Operation,
|
||||
class TupleSrc, int... Is,
|
||||
class TupleDst, int... Id>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_explode(TupleSrc&& s, int_sequence<Is...>,
|
||||
TupleDst&& d, int_sequence<Id...>)
|
||||
{
|
||||
return Operation::copy(get<Is>(static_cast<TupleSrc&&>(s))...,
|
||||
get<Id>(static_cast<TupleDst&&>(d))...);
|
||||
}
|
||||
|
||||
template <class Operation,
|
||||
class TupleAux, int... Ia,
|
||||
class TupleSrc, int... Is,
|
||||
class TupleDst, int... Id>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_explode(TupleAux&& a, int_sequence<Ia...>,
|
||||
TupleSrc&& s, int_sequence<Is...>,
|
||||
TupleDst&& d, int_sequence<Id...>)
|
||||
{
|
||||
return Operation::copy(get<Ia>(static_cast<TupleAux&&>(a))...,
|
||||
get<Is>(static_cast<TupleSrc&&>(s))...,
|
||||
get<Id>(static_cast<TupleDst&&>(d))...);
|
||||
}
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
//
|
||||
@@ -139,8 +177,8 @@ copy_unpack(Copy_Traits<CopyOp,Args...> const&,
|
||||
CUTE_STATIC_ASSERT_V(size(rD) == Int<RegNumDst>{},
|
||||
"Copy_Traits: dst failed to vectorize into registers. Layout is incompatible with this CopyOp.");
|
||||
|
||||
detail::copy_explode<CopyOp>(rS, make_int_sequence<RegNumSrc>{},
|
||||
rD, make_int_sequence<RegNumDst>{});
|
||||
detail::copy_explode_index<CopyOp>(rS, make_int_sequence<RegNumSrc>{},
|
||||
rD, make_int_sequence<RegNumDst>{});
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
@@ -0,0 +1,879 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
/*! \file
|
||||
\brief im2col make_tma_copy
|
||||
*/
|
||||
|
||||
#include "cute/arch/copy_sm90.hpp"
|
||||
#include "cute/arch/copy_sm90_desc.hpp"
|
||||
#include "cute/tensor.hpp"
|
||||
|
||||
#include "cute/algorithm/prefetch.hpp"
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
// Utility for unpacking TMA_LOAD_IM2COL arguments into a CopyOp
|
||||
template <class CopyOp>
|
||||
struct TMA_LOAD_IM2COL_Unpack
|
||||
{
|
||||
/// Copy from src to dst.
|
||||
///
|
||||
/// @param traits Copy traits created with a TMA descriptor that
|
||||
/// correctly matches the input tensor and other convolution
|
||||
/// parameters.
|
||||
///
|
||||
/// @param src Tile of the im2col-transformed coordinate tensor
|
||||
/// (result of get_tma_tensor), representing the global-memory
|
||||
/// tensor from which to load.
|
||||
///
|
||||
/// @param dst Shared memory tile, into which to load.
|
||||
template <class... Args,
|
||||
class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits<CopyOp, Args...> const& traits,
|
||||
Tensor<TS,SLayout> const& src, // tile of the transformed global activation (A) tensor
|
||||
Tensor<TD,DLayout> & dst) // shared memory tile
|
||||
{
|
||||
auto src_coord_offset = src(Int<0>{});
|
||||
auto src_coord_cwhdn_offset_srt = flatten(src_coord_offset);
|
||||
// Interpret the TMA IM2COL coordinate as (c, ([w,h,d]), n, ([s,r,t]))
|
||||
CUTE_STATIC_ASSERT_V(rank(src_coord_offset) == _4{});
|
||||
CUTE_STATIC_ASSERT_V(rank<1>(src_coord_offset) == rank<3>(src_coord_offset));
|
||||
|
||||
if constexpr (detail::is_prefetch<CopyOp>) {
|
||||
return detail::copy_explode<CopyOp>(traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
src_coord_cwhdn_offset_srt, tuple_seq<decltype(src_coord_cwhdn_offset_srt)>{});
|
||||
} else {
|
||||
static_assert(is_smem<TD>::value, "SM90_TMA_LOAD_IM2COL requires the destination be shared memory.");
|
||||
void* dst_ptr = cute::raw_pointer_cast(dst.data());
|
||||
return detail::copy_explode<CopyOp>(traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
make_tuple(dst_ptr), seq<0>{},
|
||||
src_coord_cwhdn_offset_srt, tuple_seq<decltype(src_coord_cwhdn_offset_srt)>{});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Copy_Traits for SM90 im2col TMA load comes in two layers.
|
||||
//
|
||||
// 1. Copy_Traits<SM90_TMA_LOAD_IM2COL>
|
||||
// 2. Copy_Traits<SM90_TMA_LOAD_IM2COL_OP>
|
||||
//
|
||||
// Copy_Traits<SM90_TMA_LOAD_IM2COL>
|
||||
// is the "outer" layer. It has a TMA descriptor,
|
||||
// but no barrier ("tma_mbar"), so it's "nonexecutable."
|
||||
// One calls its "with" member function with a barrier,
|
||||
// to get an executable "inner"-layer
|
||||
// Copy_Traits<SM90_TMA_LOAD_IM2COL_OP> object.
|
||||
// That object's "copy_unpack" member function
|
||||
// actually invokes im2col TMA load.
|
||||
|
||||
struct SM90_TMA_LOAD_IM2COL_OP : SM90_TMA_LOAD_IM2COL {};
|
||||
|
||||
/// @brief Non-executable specialization of Copy_Traits for SM90
|
||||
/// im2col TMA load, with TMA descriptor but no barrier.
|
||||
///
|
||||
/// Use `.with(memory_barrier)` to construct an executable version.
|
||||
template <class NumBitsPerTMA, class TMATensor>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_IM2COL, NumBitsPerTMA, TMATensor>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
Im2ColTmaDescriptor tma_desc_;
|
||||
TMATensor tma_tensor_;
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Im2ColTmaDescriptor const*
|
||||
get_tma_descriptor() const
|
||||
{
|
||||
return &tma_desc_;
|
||||
}
|
||||
|
||||
template <class GShape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
TMATensor const
|
||||
get_tma_tensor(GShape const&) const
|
||||
{
|
||||
return tma_tensor_;
|
||||
}
|
||||
|
||||
/// @brief Get an executable specialization.
|
||||
///
|
||||
/// Copy_Traits specializations with SM90_TMA_LOAD_IM2COL are not
|
||||
/// directly executable. Instead, call this "with" member function
|
||||
/// to get an executable specialization. "Executable" means that
|
||||
/// @c copy_unpack works.
|
||||
///
|
||||
/// @param tma_mbar Memory barrier for synchronization
|
||||
///
|
||||
/// @param multicast_mask Multicast mask (unused; only exists
|
||||
/// for interface compatibility with the actual multicast Copy_Traits)
|
||||
///
|
||||
/// @return Executable specialization of @c Copy_Traits
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_LOAD_IM2COL_OP, NumBitsPerTMA>
|
||||
with(uint64_t& tma_mbar, [[maybe_unused]] uint16_t const& multicast_mask = 0) const
|
||||
{
|
||||
return {{}, {&tma_desc_, &tma_mbar}};
|
||||
}
|
||||
|
||||
// Copy_Traits specializations with SM90_TMA_LOAD_IM2COL
|
||||
// are not directly executable. Instead, call .with
|
||||
// to get an executable specialization.
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst) = delete;
|
||||
};
|
||||
|
||||
/// @brief Executable specialization of Copy_Traits for SM90 im2col
|
||||
/// TMA load, with TMA descriptor and barrier.
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_IM2COL_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_IM2COL_Unpack<SM90_TMA_LOAD_IM2COL_OP>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD_IM2COL arguments
|
||||
tuple<
|
||||
Im2ColTmaDescriptor const*,
|
||||
uint64_t* // smem mbarrier
|
||||
> const opargs_;
|
||||
};
|
||||
|
||||
template <class NumBitsPerTMA, class... Args>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_IM2COL::PREFETCH, NumBitsPerTMA, Args...>
|
||||
: TMA_LOAD_IM2COL_Unpack<SM90_TMA_LOAD_IM2COL::PREFETCH>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD_IM2COL::PREFETCH arguments
|
||||
tuple<Im2ColTmaDescriptor const*> const opargs_;
|
||||
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(Copy_Traits<SM90_TMA_LOAD_IM2COL, NumBitsPerTMA, Args...> const& traits)
|
||||
: opargs_({&traits.tma_desc_}) {}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_LOAD_MULTICAST /////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct SM90_TMA_LOAD_IM2COL_MULTICAST_OP : SM90_TMA_LOAD_IM2COL_MULTICAST {};
|
||||
|
||||
/// @brief Non-executable specialization of Copy_Traits for SM90
|
||||
/// im2col TMA load, with TMA descriptor but no barrier or multicast
|
||||
/// mask.
|
||||
///
|
||||
/// Use `.with(memory_barrier)` to construct an executable version.
|
||||
template <class NumBitsPerTMA, class TMATensor>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_IM2COL_MULTICAST, NumBitsPerTMA, TMATensor>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
Im2ColTmaDescriptor tma_desc_;
|
||||
TMATensor tma_tensor_;
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Im2ColTmaDescriptor const*
|
||||
get_tma_descriptor() const {
|
||||
return &tma_desc_;
|
||||
}
|
||||
|
||||
template <class GShape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
TMATensor const
|
||||
get_tma_tensor(GShape const&) const
|
||||
{
|
||||
return tma_tensor_;
|
||||
}
|
||||
|
||||
/// @brief Get an executable specialization.
|
||||
///
|
||||
/// Copy_Traits specializations with SM90_TMA_LOAD_IM2COL_MULTICAST
|
||||
/// are not directly executable. Instead, call this "with" member
|
||||
/// function to get an executable specialization. "Executable"
|
||||
/// means that @c copy_unpack works.
|
||||
///
|
||||
/// @param tma_mbar Memory barrier for synchronization
|
||||
///
|
||||
/// @param multicast_mask Multicast mask (defaults to a single CTA)
|
||||
///
|
||||
/// @return Executable specialization of @c Copy_Traits
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_LOAD_IM2COL_MULTICAST_OP, NumBitsPerTMA>
|
||||
with(uint64_t& tma_mbar, uint16_t const& multicast_mask) const {
|
||||
return {{}, {&tma_desc_, &tma_mbar, multicast_mask}};
|
||||
}
|
||||
|
||||
// Copy_Traits specializations with SM90_TMA_LOAD_IM2COL_MULTICAST
|
||||
// are not directly executable. Instead, call .with to get an
|
||||
// executable specialization.
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst) = delete;
|
||||
};
|
||||
|
||||
/// @brief Executable specialization of Copy_Traits for SM90 multicast
|
||||
/// im2col TMA load, with TMA descriptor, barrier, and multicast mask.
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_IM2COL_MULTICAST_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_IM2COL_Unpack<SM90_TMA_LOAD_IM2COL_MULTICAST_OP>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit.
|
||||
using SrcLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1, NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD_IM2COL_MULTICAST arguments
|
||||
tuple<
|
||||
Im2ColTmaDescriptor const*,
|
||||
uint64_t*, // smem mbarrier
|
||||
uint16_t // multicast mask
|
||||
> const opargs_;
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_STORE IM2COL////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// The executable SM90_TMA_STORE_IM2COL with tma_desc
|
||||
template <class NumBitsPerTMA, class TMATensor>
|
||||
struct Copy_Traits<SM90_TMA_STORE_IM2COL, NumBitsPerTMA, TMATensor>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_STORE_IM2COL arguments
|
||||
Im2ColTmaDescriptor tma_desc_;
|
||||
TMATensor tma_tensor_;
|
||||
|
||||
// Return TmaDescriptor/TensorMap
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Im2ColTmaDescriptor const*
|
||||
get_tma_descriptor() const {
|
||||
return &tma_desc_;
|
||||
}
|
||||
|
||||
template <class GShape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
TMATensor const
|
||||
get_tma_tensor(GShape const&) const
|
||||
{
|
||||
return tma_tensor_;
|
||||
}
|
||||
|
||||
// This is the copy_unpack dispatch for this Copy_Traits
|
||||
// Src needs to be a smem tensor
|
||||
// Dst needs to be a gmem tensor with TmaCoordIterator .data()
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_STORE_IM2COL");
|
||||
|
||||
void const* const desc_ptr = &(traits.tma_desc_);
|
||||
void const* const src_ptr = cute::raw_pointer_cast(src.data());
|
||||
auto dst_coord = flatten(take<0,3>(dst(Int<0>{})));
|
||||
|
||||
return detail::copy_explode<SM90_TMA_STORE_IM2COL>(make_tuple(desc_ptr, src_ptr), seq<0,1>{},
|
||||
dst_coord, tuple_seq<decltype(dst_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
|
||||
/// @brief Creates a TMA descriptor for im2col TMA load.
|
||||
///
|
||||
/// @param tensor_cwhdn Global activation tensor (A matrix of Fprop).
|
||||
/// This is the original (not im2col-transformed) tensor in global
|
||||
/// memory.
|
||||
///
|
||||
/// @param slayout Rank 2 (M,K) shared memory layout of the activation
|
||||
/// tensor. Here, K is "GEMM K," not the filter tensor's mode of
|
||||
/// the same name.
|
||||
//////
|
||||
/// @param traversal_stride Traversal strides convolution parameter
|
||||
//////
|
||||
/// Each of padding_shape, traversal_stride, and dilation_shape is a
|
||||
/// tuple whose size is the number of spatial modes (e.g., 3 for a 5-D
|
||||
/// convolution).
|
||||
///
|
||||
/// @return TMA descriptor for im2col TMA load
|
||||
template <class EngineA, class LayoutA,
|
||||
class SmemSwizzle, class TMALayout,
|
||||
class LowerCornerStride,
|
||||
class UpperCornerStride,
|
||||
class LowerPaddingStride,
|
||||
class UpperPaddingStride,
|
||||
class TraversalStride,
|
||||
class LowerSRTStride,
|
||||
class DilationStride>
|
||||
CUTE_HOST
|
||||
auto
|
||||
make_im2col_tma_copy_desc(
|
||||
Tensor<EngineA, LayoutA> const& tensor_cwhdn, // (C,W,H,D,N)
|
||||
uint32_t range_c, // TILE_C
|
||||
uint32_t range_whdn, // TILE_WHDN
|
||||
SmemSwizzle const& smem_swizzle, // Swizzle
|
||||
TMALayout const& tma_layout_vt, // TMA layout
|
||||
LowerCornerStride const& lower_corner_whd, // WHD offset of the "base pointer"
|
||||
UpperCornerStride const& upper_corner_whd, // WHD upper corner
|
||||
LowerPaddingStride const& lower_padding_whd, // WHD lower padding
|
||||
UpperPaddingStride const& upper_padding_whd, // WHD upper padding
|
||||
TraversalStride const& stride_whd, // WHD traversal stride
|
||||
LowerSRTStride const& lower_srt, // SRT offset of the "base pointer"
|
||||
DilationStride const& stride_srt) // SRT stride - dilation
|
||||
{
|
||||
static_assert(is_gmem<EngineA>::value, "Tensor must point to GPU global memory.");
|
||||
using value_type = typename EngineA::value_type;
|
||||
|
||||
constexpr uint32_t num_total_modes = LayoutA::rank;
|
||||
constexpr int num_spatial_modes = num_total_modes - 2;
|
||||
|
||||
// Gmem starting address
|
||||
void* gmem_address = (void*) raw_pointer_cast(tensor_cwhdn.data());
|
||||
|
||||
// Gmem extents are just the tensor shape
|
||||
cute::array<uint64_t, 5> gmem_prob_shape = {1,1,1,1,1};
|
||||
for_each(make_seq<num_total_modes>{}, [&](auto i) {
|
||||
gmem_prob_shape[i] = static_cast<uint64_t>(shape<i>(tensor_cwhdn));
|
||||
});
|
||||
|
||||
// Gmem strides are byte strides of the activation tensor in CWHDN order
|
||||
cute::array<uint64_t, 5> gmem_prob_stride = {0,0,0,0,0};
|
||||
for_each(make_seq<num_total_modes>{}, [&](auto i) {
|
||||
gmem_prob_stride[i] = sizeof(value_type) * stride<i>(tensor_cwhdn);
|
||||
});
|
||||
|
||||
// Traversal strides are a function of the dilation shape
|
||||
// corresponding to spatial (WHD) modes.
|
||||
cute::array<uint32_t, 5> tma_traversal_strides = {1,1,1,1,1};
|
||||
for_each(make_seq<num_spatial_modes>{}, [&](auto i) {
|
||||
tma_traversal_strides[i+1] = static_cast<uint32_t>(get<i>(stride_whd));
|
||||
});
|
||||
|
||||
cute::array<int32_t, num_spatial_modes> tma_lower_corner{};
|
||||
for_each(make_seq<num_spatial_modes>{}, [&](auto i) {
|
||||
tma_lower_corner[i] = static_cast<int32_t>(get<i>(lower_corner_whd));
|
||||
});
|
||||
|
||||
cute::array<int32_t, num_spatial_modes> tma_upper_corner{};
|
||||
for_each(make_seq<num_spatial_modes>{}, [&](auto i) {
|
||||
tma_upper_corner[i] = static_cast<int32_t>(get<i>(upper_corner_whd));
|
||||
});
|
||||
|
||||
Im2ColTmaDescriptor tma_desc;
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 12)
|
||||
|
||||
CUtensorMapDataType tma_format = TMA::to_CUtensorMapDataType<value_type>();
|
||||
CUtensorMapInterleave tma_interleave = CU_TENSOR_MAP_INTERLEAVE_NONE;
|
||||
CUtensorMapL2promotion tma_l2Promotion = CU_TENSOR_MAP_L2_PROMOTION_NONE;
|
||||
CUtensorMapFloatOOBfill tma_oob_fill = CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE;
|
||||
CUtensorMapSwizzle tma_swizzle = TMA::to_CUtensorMapSwizzle(detail::get_tma_swizzle_bits(smem_swizzle));
|
||||
|
||||
CUresult encode_result = cuTensorMapEncodeIm2col(
|
||||
&tma_desc,
|
||||
tma_format,
|
||||
num_total_modes,
|
||||
gmem_address,
|
||||
gmem_prob_shape.data(),
|
||||
gmem_prob_stride.data() + 1, // gmem_prob_stride[0] implicitly sizeof(value_type)
|
||||
tma_lower_corner.data(),
|
||||
tma_upper_corner.data(),
|
||||
range_c,
|
||||
range_whdn,
|
||||
tma_traversal_strides.data(),
|
||||
tma_interleave,
|
||||
tma_swizzle,
|
||||
tma_l2Promotion,
|
||||
tma_oob_fill);
|
||||
|
||||
// The extra asserts help indicate the error's cause.
|
||||
assert(encode_result != CUDA_ERROR_DEINITIALIZED);
|
||||
assert(encode_result != CUDA_ERROR_NOT_INITIALIZED);
|
||||
assert(encode_result != CUDA_ERROR_INVALID_CONTEXT);
|
||||
assert(encode_result != CUDA_ERROR_INVALID_VALUE);
|
||||
assert(encode_result == CUDA_SUCCESS);
|
||||
|
||||
#endif // (__CUDACC_VER_MAJOR__ >= 12)
|
||||
//
|
||||
// Calculate gemm shapes and linearized shapes based on tma layout tiling.
|
||||
//
|
||||
|
||||
// Compute [w, h, d, n]
|
||||
// q/p/z = (w/h/d + (upper_corner_whd - lower_corner_whd - 1)) / stride_whd + 1
|
||||
auto gemm_mn_ = cute::transform(cute::make_seq<num_spatial_modes>{}, [&](auto i) {
|
||||
return (shape<i+1>(tensor_cwhdn) + get<i>(upper_corner_whd) - get<i>(lower_corner_whd) - Int<1>{}) / get<i>(stride_whd) + Int<1>{};
|
||||
});
|
||||
auto gemm_mn = append(gemm_mn_, shape<num_spatial_modes+1>(tensor_cwhdn));
|
||||
|
||||
// Compute [c, s, r, t]
|
||||
// fprop/wgrad, s/r/t = 1 + (upper_padding_whd - upper_corner_whd) / stride_srt
|
||||
// wgrad, s/r/t = 1 + (lower_padding_whd - lower_corner_whd) / stride_srt
|
||||
auto gemm_k_ = cute::transform(cute::make_seq<num_spatial_modes>{}, [&](auto i) {
|
||||
auto padding_size = conditional_return(get<i>(stride_srt) > Int<0>{},
|
||||
get<i>(upper_padding_whd) - get<i>(upper_corner_whd),
|
||||
get<i>(lower_corner_whd) - get<i>(lower_padding_whd));
|
||||
return Int<1>{} + padding_size / get<i>(stride_srt);
|
||||
});
|
||||
auto gemm_k = prepend(gemm_k_, shape<0>(tensor_cwhdn));
|
||||
|
||||
// For fprop/dgrad kernel, gemm_shapes is ((q, p, z, n), (c, s, r, t))
|
||||
// For wgrad kernel, gemm_shapes is ((c, s, r, t), (q, p, z, n))
|
||||
auto gemm_shapes_common = make_shape(gemm_mn, gemm_k);
|
||||
auto gemm_shapes = make_shape(
|
||||
basis_get(stride<0,1>(tma_layout_vt), gemm_shapes_common),
|
||||
basis_get(stride<0,0>(tma_layout_vt), gemm_shapes_common));
|
||||
|
||||
// For fprop/dgrad kernel, linearized shapes is (whdn, (c, s, r, t))
|
||||
// For wgrad kernel linearized shapes is ((c, s, r, t), whdn)
|
||||
auto linear_shapes_common = make_shape(size(gemm_mn), gemm_k);
|
||||
auto linear_shapes = make_shape(
|
||||
basis_get(stride<0,1>(tma_layout_vt), linear_shapes_common),
|
||||
basis_get(stride<0,0>(tma_layout_vt), linear_shapes_common));
|
||||
|
||||
//
|
||||
// Calculate gmem basis stride based on tma layout tiling.
|
||||
//
|
||||
|
||||
auto tma_basis_scale = make_shape(Int<1>{}, stride_whd, Int<1>{}, stride_srt);
|
||||
auto tma_basis = elem_scale(tma_basis_scale, make_basis_like(tma_basis_scale));
|
||||
|
||||
auto gbasis_strides_common = make_stride(
|
||||
append(get<1>(tma_basis), get<2>(tma_basis)),
|
||||
prepend(get<3>(tma_basis), get<0>(tma_basis))); // ((w,h,d,n),(c,s,r,t))
|
||||
auto gbasis_strides = make_stride(
|
||||
basis_get(stride<0,1>(tma_layout_vt), gbasis_strides_common),
|
||||
basis_get(stride<0,0>(tma_layout_vt), gbasis_strides_common));
|
||||
|
||||
//
|
||||
// Create tma tensor
|
||||
//
|
||||
|
||||
auto lower_corner = make_arithmetic_tuple(Int<0>{}, lower_corner_whd, Int<0>{}, lower_srt);
|
||||
|
||||
auto tensor_multimode = make_tensor(ArithmeticTupleIterator(lower_corner), gemm_shapes, gbasis_strides);
|
||||
auto tensor_linear = make_identity_tensor(linear_shapes);
|
||||
auto tma_tensor = make_tensor(tensor_multimode.data(), composition(
|
||||
tensor_multimode.layout(),
|
||||
tensor_linear(Int<0>{}),
|
||||
tensor_linear.layout()));
|
||||
|
||||
return cute::make_tuple(tma_desc, tma_tensor);
|
||||
}
|
||||
|
||||
/// Make a TiledCopy for im2col TMA load.
|
||||
///
|
||||
/// @param copy_op The copy implementation: either
|
||||
/// SM90_TMA_LOAD_IM2COL or SM90_TMA_LOAD_IM2COL_MULTICAST.
|
||||
///
|
||||
/// @param tensor_cwhdn The global tensor to use for im2col TMA loads.
|
||||
/// For Fprop convolutions, this is the activation tensor. This is
|
||||
/// the "original tensor that points to global memory, not the
|
||||
/// coordinate (im2col-transformed) tensor.
|
||||
///
|
||||
/// @param slayout Layout of shared memory tile.
|
||||
///
|
||||
/// @param stride_whd The traversal strides convolution
|
||||
/// parameter.
|
||||
///
|
||||
/// @return TiledCopy specialization for im2col TMA loads.
|
||||
template <class CopyOp,
|
||||
class GEngine, class GLayout,
|
||||
class SLayout,
|
||||
class TShape, class TStride,
|
||||
class VShape, class VStride,
|
||||
class LowerCornerStride,
|
||||
class UpperCornerStride,
|
||||
class LowerPaddingStride,
|
||||
class UpperPaddingStride,
|
||||
class TraversalStride,
|
||||
class LowerSRTStride,
|
||||
class DilationStride>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_tma_copy_im2col(CopyOp const& copy_op,
|
||||
Tensor<GEngine,GLayout> const& gtensor,
|
||||
SLayout const& slayout,
|
||||
Layout<TShape,TStride> const& cta_t_map, // CTA tid -> logical TMA tid
|
||||
Layout<VShape,VStride> const& cta_v_map, // CTA vid -> gmem coord
|
||||
LowerCornerStride const& lower_corner_whd,
|
||||
UpperCornerStride const& upper_corner_whd,
|
||||
LowerPaddingStride const& lower_padding_whd,
|
||||
UpperPaddingStride const& upper_padding_whd,
|
||||
TraversalStride const& stride_whd, // traversal stride
|
||||
LowerSRTStride const& lower_srt,
|
||||
DilationStride const& stride_srt) // dilation
|
||||
{
|
||||
//
|
||||
// TMA parameter checking
|
||||
//
|
||||
|
||||
CUTE_STATIC_ASSERT_V(product_each(shape(slayout)) == product_each(shape(cta_v_map)),
|
||||
"TMA requires CTA_Tile and SLayout top-level shape equivalence.");
|
||||
CUTE_STATIC_ASSERT_V(size(slayout) % cosize(cta_t_map) == Int<0>{},
|
||||
"Number of active CTAs in TMA must divide domain size of slayout.");
|
||||
|
||||
//
|
||||
// TMA slayout manipulation
|
||||
//
|
||||
|
||||
// Invert the smem to get the largest contiguous vector in the smem layout
|
||||
auto inv_smem_layout = right_inverse(get_nonswizzle_portion(slayout));
|
||||
// trunc_smem_idx -> trunc_smem_coord
|
||||
|
||||
// Map from smem idx to a gmem mode
|
||||
auto sidx_to_gmode = coalesce(composition(cta_v_map, inv_smem_layout));
|
||||
|
||||
#if 0
|
||||
print("g_layout : "); print(gtensor.layout()); print("\n");
|
||||
print("s_layout : "); print(slayout); print("\n");
|
||||
print("cta_t_map : "); print(cta_t_map); print("\n");
|
||||
print("cta_v_map : "); print(cta_v_map); print("\n");
|
||||
print("inv_smem : "); print(inv_smem_layout); print("\n");
|
||||
print("sidx_to_gmode : "); print(sidx_to_gmode); print("\n");
|
||||
#endif
|
||||
|
||||
//
|
||||
// TMA gtensor manipulation
|
||||
//
|
||||
|
||||
// Generate a TupleBasis for the gtensor
|
||||
auto glayout_basis = make_identity_layout(product_each(shape(gtensor)));
|
||||
|
||||
// Tile the modes of gtensor with the truncated cta_v_map o inv_smem_layout_trunc
|
||||
auto tma_layout_full = flatten(composition(glayout_basis, sidx_to_gmode));
|
||||
|
||||
// Truncate any incompatibilities -- no starting in the middle of gmodes
|
||||
auto smem_rank = find_if(stride(tma_layout_full), [](auto e) {
|
||||
[[maybe_unused]] auto v = basis_value(e);
|
||||
return not is_constant<1,decltype(v)>{};
|
||||
});
|
||||
static_assert(smem_rank >= 2, "IM2COL expects at least 2 modes of the smem to vectorize with gmem.");
|
||||
// IM2COL uses a maximum of 2 modes
|
||||
constexpr int smem_tma_rank = cute::min(int(smem_rank), 2);
|
||||
|
||||
// Keep only the static-1 basis modes into gmem
|
||||
auto tma_layout_trunc = take<0,smem_tma_rank>(tma_layout_full);
|
||||
|
||||
// Split according to the portion each multicast CTA will be responsible for
|
||||
auto tma_layout_vt = logical_divide(tma_layout_trunc, shape_div(size(tma_layout_trunc), cosize(cta_t_map)));
|
||||
|
||||
#if 0
|
||||
print("glayout_basis : "); print(glayout_basis); print("\n");
|
||||
print("tma_layout_full : "); print(tma_layout_full); print("\n");
|
||||
|
||||
print("tma_layout_trunc: "); print(tma_layout_trunc); print("\n");
|
||||
print("tma_layout_vt : "); print(tma_layout_vt); print("\n");
|
||||
#endif
|
||||
|
||||
auto range_c = size<0,0>(tma_layout_vt);
|
||||
auto range_whdn = size<0,1>(tma_layout_vt);
|
||||
|
||||
Tensor gtensor_cwhdn = make_tensor(gtensor.data(),
|
||||
flatten(make_layout(basis_get(stride<0,0>(tma_layout_vt), gtensor.layout()),
|
||||
basis_get(stride<0,1>(tma_layout_vt), gtensor.layout()))));
|
||||
|
||||
auto [tma_desc, tma_tensor] = make_im2col_tma_copy_desc(
|
||||
gtensor_cwhdn,
|
||||
range_c,
|
||||
range_whdn,
|
||||
detail::get_swizzle_portion(slayout),
|
||||
tma_layout_vt,
|
||||
lower_corner_whd,
|
||||
upper_corner_whd,
|
||||
lower_padding_whd,
|
||||
upper_padding_whd,
|
||||
stride_whd,
|
||||
lower_srt,
|
||||
stride_srt);
|
||||
|
||||
//
|
||||
// Construct the Copy_Traits
|
||||
//
|
||||
|
||||
using T = typename GEngine::value_type;
|
||||
constexpr int num_bits_per_tma = decltype(size<0>(tma_layout_vt))::value * sizeof(T) * 8;
|
||||
|
||||
using Traits = Copy_Traits<CopyOp, cute::C<num_bits_per_tma>, decltype(tma_tensor)>;
|
||||
|
||||
#if 0
|
||||
print("num_bits : "); print(NumBitsPerTMA{}); print("\n");
|
||||
#endif
|
||||
|
||||
Traits tma_traits{tma_desc, tma_tensor};
|
||||
|
||||
//
|
||||
// Construct the TiledCopy
|
||||
//
|
||||
|
||||
auto cta_tiler = product_each(shape(cta_v_map));
|
||||
|
||||
// (CTA V, CTA T) -> smem_coord
|
||||
auto layout_vt = composition(inv_smem_layout, make_layout(shape(tma_layout_vt)));
|
||||
// Scale that up to cover all of the smem_coords
|
||||
//
|
||||
// The smem vector might not cover all of the tile,
|
||||
// so multiply it up to cover the entire tile.
|
||||
// "T" here (the parallel index) is a CTA index.
|
||||
auto layout_VT = tile_to_shape(layout_vt, make_shape(size(cta_v_map)/size<1>(layout_vt), size<1>(layout_vt)));
|
||||
// Flip it and change the domain of the T from logical thr to thr_idx
|
||||
auto layout_TV = make_layout(composition(layout<1>(layout_VT), cta_t_map), layout<0>(layout_VT));
|
||||
|
||||
#if 0
|
||||
print("cta_tiler : "); print(cta_tiler); print("\n");
|
||||
print("layout_VT : "); print(layout_VT); print("\n");
|
||||
print("layout_TV : "); print(layout_TV); print("\n");
|
||||
#endif
|
||||
|
||||
using T = typename GEngine::value_type;
|
||||
return TiledCopy<Copy_Atom<Traits,T>, decltype(layout_TV), decltype(cta_tiler)>{tma_traits};
|
||||
}
|
||||
|
||||
/// Make a TiledCopy for im2col TMA with no offsets.
|
||||
/// E.g. im2col TMA load for C and im2col TMA store for D.
|
||||
template <class CopyOp,
|
||||
class GEngine, class GLayout,
|
||||
class SLayout,
|
||||
class TShape, class TStride,
|
||||
class VShape, class VStride>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_tma_copy_im2col(CopyOp const& copy_op,
|
||||
Tensor<GEngine,GLayout> const& gtensor,
|
||||
SLayout const& slayout,
|
||||
Layout<TShape,TStride> const& cta_t_map, // CTA tid -> logical TMA tid
|
||||
Layout<VShape,VStride> const& cta_v_map) // CTA vid -> gmem coord
|
||||
{
|
||||
constexpr int num_spatial_modes = rank<0>(GLayout{}) - 1;
|
||||
return make_tma_copy_im2col(copy_op, gtensor, slayout, cta_t_map, cta_v_map,
|
||||
append<num_spatial_modes>(Stride<_0>{}, Int<0>{}), // lower_corner_whd
|
||||
append<num_spatial_modes>(Stride<_0>{}, Int<0>{}), // upper_corner_whd
|
||||
append<num_spatial_modes>(Stride<_0>{}, Int<0>{}), // lower_padding_whd
|
||||
append<num_spatial_modes>(Stride<_0>{}, Int<0>{}), // upper_padding_whd
|
||||
append<num_spatial_modes>(Stride<_1>{}, Int<1>{}), // stride_whd
|
||||
append<num_spatial_modes>(Stride<_0>{}, Int<0>{}), // lower_srt
|
||||
append<num_spatial_modes>(Stride<_1>{}, Int<1>{})); // stride_srt
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
|
||||
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout,
|
||||
class CTATiler,
|
||||
class MulticastSize,
|
||||
class LowerCornerStride,
|
||||
class UpperCornerStride,
|
||||
class LowerPaddingStride,
|
||||
class UpperPaddingStride,
|
||||
class TraversalStride,
|
||||
class LowerSRTStride,
|
||||
class DilationStride>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout,
|
||||
CTATiler const& cta_tiler,
|
||||
MulticastSize const& multicast_size,
|
||||
LowerCornerStride const& lower_corner_whd,
|
||||
UpperCornerStride const& upper_corner_whd,
|
||||
LowerPaddingStride const& lower_padding_whd,
|
||||
UpperPaddingStride const& upper_padding_whd,
|
||||
TraversalStride const& stride_whd,
|
||||
LowerSRTStride const& lower_srt,
|
||||
DilationStride const& stride_srt)
|
||||
{
|
||||
auto cta_v_tile = make_identity_layout(product_each(shape(tensor_cwhdn))).compose(cta_tiler);
|
||||
auto cta_t_tile = make_layout(multicast_size);
|
||||
|
||||
return detail::make_tma_copy_im2col(copy_op, tensor_cwhdn,
|
||||
slayout, cta_t_tile, cta_v_tile,
|
||||
lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt);
|
||||
}
|
||||
|
||||
// Explicit default for multicast_size
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout,
|
||||
class CTATiler,
|
||||
class LowerCornerStride,
|
||||
class UpperCornerStride,
|
||||
class LowerPaddingStride,
|
||||
class UpperPaddingStride,
|
||||
class TraversalStride,
|
||||
class LowerSRTStride,
|
||||
class DilationStride>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout,
|
||||
CTATiler const& cta_tiler,
|
||||
LowerCornerStride const& lower_corner_whd,
|
||||
UpperCornerStride const& upper_corner_whd,
|
||||
LowerPaddingStride const& lower_padding_whd,
|
||||
UpperPaddingStride const& upper_padding_whd,
|
||||
TraversalStride const& stride_whd,
|
||||
LowerSRTStride const& lower_srt,
|
||||
DilationStride const& stride_srt)
|
||||
{
|
||||
return make_im2col_tma_copy(copy_op, tensor_cwhdn, slayout, cta_tiler, Int<1>{},
|
||||
lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt);
|
||||
}
|
||||
|
||||
// Explicit default for cta_tiler and multicast_size
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout,
|
||||
class LowerCornerStride,
|
||||
class UpperCornerStride,
|
||||
class LowerPaddingStride,
|
||||
class UpperPaddingStride,
|
||||
class TraversalStride,
|
||||
class LowerSRTStride,
|
||||
class DilationStride>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout,
|
||||
LowerCornerStride const& lower_corner_whd,
|
||||
UpperCornerStride const& upper_corner_whd,
|
||||
LowerPaddingStride const& lower_padding_whd,
|
||||
UpperPaddingStride const& upper_padding_whd,
|
||||
TraversalStride const& stride_whd,
|
||||
LowerSRTStride const& lower_srt,
|
||||
DilationStride const& stride_srt)
|
||||
{
|
||||
return make_im2col_tma_copy(copy_op, tensor_cwhdn, slayout, product_each(shape(slayout)), Int<1>{},
|
||||
lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt);
|
||||
}
|
||||
|
||||
// No offsets copy.
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout,
|
||||
class CTATiler,
|
||||
class MulticastSize>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout,
|
||||
CTATiler const& cta_tiler,
|
||||
MulticastSize const& multicast_size)
|
||||
{
|
||||
auto cta_v_tile = make_identity_layout(product_each(shape(tensor_cwhdn))).compose(cta_tiler);
|
||||
auto cta_t_tile = make_layout(multicast_size);
|
||||
|
||||
return detail::make_tma_copy_im2col(copy_op, tensor_cwhdn, slayout, cta_t_tile, cta_v_tile);
|
||||
}
|
||||
|
||||
// Explicit default for multicast_size
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout,
|
||||
class CTATiler>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout,
|
||||
CTATiler const& cta_tiler)
|
||||
{
|
||||
return make_im2col_tma_copy(copy_op, tensor_cwhdn, slayout, cta_tiler, Int<1>{});
|
||||
}
|
||||
|
||||
// Explicit default for cta_tiler and multicast_size
|
||||
template <class CopyOp,
|
||||
class Engine0, class Layout0,
|
||||
class SLayout>
|
||||
CUTE_HOST_RTC
|
||||
auto
|
||||
make_im2col_tma_copy(CopyOp const& copy_op,
|
||||
Tensor<Engine0, Layout0> const& tensor_cwhdn,
|
||||
SLayout const& slayout)
|
||||
{
|
||||
return make_im2col_tma_copy(copy_op, tensor_cwhdn, slayout, product_each(shape(slayout)), Int<1>{});
|
||||
}
|
||||
|
||||
} // namespace cute
|
||||
@@ -38,6 +38,8 @@
|
||||
#include <cute/atom/copy_traits.hpp>
|
||||
#include <cute/atom/copy_atom.hpp>
|
||||
|
||||
#include <cute/algorithm/prefetch.hpp>
|
||||
|
||||
#include <cute/numeric/integral_ratio.hpp>
|
||||
|
||||
namespace cute
|
||||
@@ -53,77 +55,55 @@ struct AuxTmaParams {
|
||||
static_assert(is_static<TmaSwizzle>::value);
|
||||
};
|
||||
|
||||
// Utility for unpacking TMA_LOAD arguments into a CopyOp
|
||||
template <class CopyOp>
|
||||
struct TMA_LOAD_Unpack
|
||||
{
|
||||
template <class... Args,
|
||||
class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits<CopyOp, Args...> const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
auto src_coord = src.data().coord_;
|
||||
if constexpr (detail::is_prefetch<CopyOp>) {
|
||||
return detail::copy_explode<CopyOp>(traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
} else {
|
||||
static_assert(is_smem<TD>::value, "SM90_TMA_LOAD requires the destination be shared memory.");
|
||||
void* dst_ptr = cute::raw_pointer_cast(dst.data());
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(src_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), dst_ptr);
|
||||
#endif
|
||||
return detail::copy_explode<CopyOp>(traits.opargs_, tuple_seq<decltype(traits.opargs_)>{},
|
||||
make_tuple(dst_ptr), seq<0>{},
|
||||
src_coord, tuple_seq<decltype(src_coord)>{});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_LOAD ///////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct SM90_TMA_LOAD_OP : SM90_TMA_LOAD {};
|
||||
|
||||
// The executable SM90_TMA_LOAD with tma_desc and tma_mbar
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD arguments
|
||||
TmaDescriptor const& tma_desc_;
|
||||
uint64_t& tma_load_mbar_;
|
||||
|
||||
template <class Coord, int... Is>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_unpack_(void const* const dst_ptr,
|
||||
Coord const& src_coord, seq<Is...>) const
|
||||
{
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(src_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), dst_ptr);
|
||||
#endif
|
||||
|
||||
SM90_TMA_LOAD::copy(&tma_desc_, tma_load_mbar_,
|
||||
dst_ptr, get<Is>(src_coord)...);
|
||||
}
|
||||
|
||||
// This is the copy_unpack dispatch for this Copy_Traits
|
||||
// Src needs to be a gmem tensor with TmaCoordIterator .data()
|
||||
// Dst needs to be a smem tensor
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr
|
||||
void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TD>::value, "Expected smem dst for SM90_TMA_LOAD");
|
||||
|
||||
traits.copy_unpack_(cute::raw_pointer_cast(dst.data()), src.data().coord_, tuple_seq<decltype(src.data().coord_)>{});
|
||||
}
|
||||
};
|
||||
|
||||
// The non-executable SM90_TMA_LOAD with tma_desc and no tma_mbar
|
||||
// Use .with(tma_mbar) to construct an executable version
|
||||
template <class NumBitsPerTMA, class AuxParams_>
|
||||
struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
@@ -144,7 +124,7 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
with(uint64_t& tma_mbar, [[maybe_unused]] uint16_t const& multicast_mask = 0) const {
|
||||
// We accept multicast_mask here to keep the API for both atoms consistent
|
||||
return {tma_desc_, tma_mbar};
|
||||
return {{}, {&tma_desc_, &tma_mbar}};
|
||||
}
|
||||
|
||||
// Construct an executable SM90_TMA_LOAD with tma_mbar (temp. overloaded for grouped gemm/ptr array gemm)
|
||||
@@ -152,7 +132,7 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
with(TmaDescriptor const* new_tma_desc, uint64_t& tma_mbar, [[maybe_unused]] uint16_t const& multicast_mask = 0) const {
|
||||
// We accept multicast_mask here to keep the API for both atoms consistent
|
||||
return {*new_tma_desc, tma_mbar};
|
||||
return {{}, {new_tma_desc, &tma_mbar}};
|
||||
}
|
||||
|
||||
// Generate the TMA coord tensor
|
||||
@@ -173,72 +153,65 @@ struct Copy_Traits<SM90_TMA_LOAD, NumBitsPerTMA, AuxParams_>
|
||||
Tensor<TD,DLayout> & dst) = delete;
|
||||
};
|
||||
|
||||
// The executable SM90_TMA_LOAD with tma_desc and tma_mbar
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_OP>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD arguments
|
||||
tuple<
|
||||
TmaDescriptor const*,
|
||||
uint64_t* // smem mbarrier
|
||||
> const opargs_;
|
||||
};
|
||||
|
||||
// The prefetch for SM90_TMA_LOAD with tma_desc
|
||||
template <class NumBitsPerTMA, class... Args>
|
||||
struct Copy_Traits<SM90_TMA_LOAD::PREFETCH, NumBitsPerTMA, Args...>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD::PREFETCH>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD::PREFETCH arguments
|
||||
tuple<TmaDescriptor const*> const opargs_;
|
||||
|
||||
// Construct with any other Traits' TMA Desc
|
||||
template <class... CopyArgs>
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(Copy_Traits<CopyArgs...> const& traits)
|
||||
: opargs_({&traits.tma_desc_}) {}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_LOAD_MULTICAST /////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct SM90_TMA_LOAD_MULTICAST_OP : SM90_TMA_LOAD_MULTICAST {};
|
||||
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD_MULTICAST arguments
|
||||
TmaDescriptor const& tma_desc_;
|
||||
uint64_t& tma_load_mbar_;
|
||||
uint16_t const& multicast_mask_;
|
||||
|
||||
template <class Coord, int... Is>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
copy_unpack_(void const* const dst_ptr,
|
||||
Coord const& src_coord, seq<Is...>) const
|
||||
{
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(src_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), dst_ptr);
|
||||
#endif
|
||||
|
||||
SM90_TMA_LOAD_MULTICAST::copy(&tma_desc_, tma_load_mbar_, multicast_mask_,
|
||||
dst_ptr, get<Is>(src_coord)...);
|
||||
}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr
|
||||
void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TD>::value, "Expected smem dst for SM90_TMA_LOAD_MULTICAST");
|
||||
|
||||
traits.copy_unpack_(cute::raw_pointer_cast(dst.data()), src.data().coord_, tuple_seq<decltype(src.data().coord_)>{});
|
||||
}
|
||||
};
|
||||
|
||||
// The non-executable SM90_TMA_LOAD_MULTICAST with tma_desc and no tma_mbar
|
||||
// Use .with(tma_mbar, multicast_mask) to construct an executable version
|
||||
template <class NumBitsPerTMA, class AuxParams_>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
@@ -258,14 +231,14 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
with(uint64_t& tma_load_mbar, uint16_t const& multicast_mask) const {
|
||||
return {tma_desc_, tma_load_mbar, multicast_mask};
|
||||
return {{}, {&tma_desc_, &tma_load_mbar, multicast_mask}};
|
||||
}
|
||||
|
||||
// Construct an executable SM90_TMA_LOAD_MULTICAST_OP with tma_mbar (temp. overloaded for grouped gemm/ptr array gemm)
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
with(TmaDescriptor const* new_tma_desc, uint64_t& tma_load_mbar, uint16_t const& multicast_mask) const {
|
||||
return {*new_tma_desc, tma_load_mbar, multicast_mask};
|
||||
return {{}, {new_tma_desc, &tma_load_mbar, multicast_mask}};
|
||||
}
|
||||
|
||||
// Generate the TMA coord tensor
|
||||
@@ -286,6 +259,27 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
Tensor<TD,DLayout> & dst) = delete;
|
||||
};
|
||||
|
||||
// The executable SM90_TMA_LOAD_MULTICAST with tma_desc and tma_mbar and multicast_mask
|
||||
template <class NumBitsPerTMA>
|
||||
struct Copy_Traits<SM90_TMA_LOAD_MULTICAST_OP, NumBitsPerTMA>
|
||||
: TMA_LOAD_Unpack<SM90_TMA_LOAD_MULTICAST_OP>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_LOAD_MULTICAST arguments
|
||||
tuple<
|
||||
TmaDescriptor const*,
|
||||
uint64_t*, // smem mbarrier
|
||||
uint16_t // multicast mask
|
||||
> const opargs_;
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_STORE //////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
@@ -293,6 +287,68 @@ struct Copy_Traits<SM90_TMA_LOAD_MULTICAST, NumBitsPerTMA, AuxParams_>
|
||||
// The executable SM90_TMA_STORE with tma_desc
|
||||
template <class NumBitsPerTMA, class AuxParams_>
|
||||
struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
// Map from (src-thr,src-val) to bit
|
||||
using SrcLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Map from (dst-thr,dst-val) to bit
|
||||
using DstLayout = Layout<Shape<_1,NumBitsPerTMA>>;
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_STORE arguments
|
||||
TmaDescriptor tma_desc_;
|
||||
using AuxParams = AuxParams_;
|
||||
AuxParams aux_params_;
|
||||
|
||||
// Return TmaDescriptor/TensorMap
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
TmaDescriptor const*
|
||||
get_tma_descriptor() const {
|
||||
return &tma_desc_;
|
||||
}
|
||||
|
||||
// Generate the TMA coord tensor
|
||||
template <class GShape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
get_tma_tensor(GShape const& g_shape) const {
|
||||
static_assert(is_congruent<decltype(g_shape), decltype(aux_params_.g_stride_)>::value);
|
||||
return make_counting_tensor(make_layout(g_shape, aux_params_.g_stride_));
|
||||
}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_STORE");
|
||||
//static_assert(is_gmem<TD>::value, "Expected gmem dst for SM90_TMA_STORE"); // TMA spoofed src tensor
|
||||
|
||||
void const* const desc_ptr = &(traits.tma_desc_);
|
||||
void const* const src_ptr = cute::raw_pointer_cast(src.data());
|
||||
auto dst_coord = dst.data().coord_;
|
||||
#if 0
|
||||
auto [c0,c1,c2,c3,c4] = append<5>(dst_coord, 0);
|
||||
printf("THR (%d,%d,%d) BLK (%d,%d,%d) TMACRD (%d,%d,%d,%d,%d) SMEMADDR (%p)\n",
|
||||
threadIdx.x, threadIdx.y, threadIdx.z,
|
||||
blockIdx.x, blockIdx.y, blockIdx.z,
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), src_ptr);
|
||||
#endif
|
||||
return detail::copy_explode<SM90_TMA_STORE>(make_tuple(desc_ptr, src_ptr), seq<0,1>{},
|
||||
dst_coord, tuple_seq<decltype(dst_coord)>{});
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////// TMA_REDUCE_ADD //////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// The executable SM90_TMA_REDUCE_ADD with tma_desc
|
||||
template <class NumBitsPerTMA, class AuxParams_>
|
||||
struct Copy_Traits<SM90_TMA_REDUCE_ADD, NumBitsPerTMA, AuxParams_>
|
||||
{
|
||||
using ThrID = Layout<_1>;
|
||||
|
||||
@@ -304,7 +360,7 @@ struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
// Reference map from (thr,val) to bit
|
||||
using RefLayout = SrcLayout;
|
||||
|
||||
// SM90_TMA_STORE arguments
|
||||
// SM90_TMA_REDUCE_ADD arguments
|
||||
TmaDescriptor tma_desc_;
|
||||
using AuxParams = AuxParams_;
|
||||
AuxParams aux_params_;
|
||||
@@ -339,7 +395,7 @@ struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
int32_t(c0), int32_t(c1), int32_t(c2), int32_t(c3), int32_t(c4), src_ptr);
|
||||
#endif
|
||||
|
||||
SM90_TMA_STORE::copy(&tma_desc_,
|
||||
SM90_TMA_REDUCE_ADD::copy(&tma_desc_,
|
||||
src_ptr, get<Is>(dst_coord)...);
|
||||
}
|
||||
|
||||
@@ -354,8 +410,8 @@ struct Copy_Traits<SM90_TMA_STORE, NumBitsPerTMA, AuxParams_>
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_STORE");
|
||||
//static_assert(is_gmem<TD>::value, "Expected gmem dst for SM90_TMA_STORE"); // TMA spoofed src tensor
|
||||
static_assert(is_smem<TS>::value, "Expected smem src for SM90_TMA_REDUCE_ADD");
|
||||
//static_assert(is_gmem<TD>::value, "Expected gmem dst for SM90_TMA_REDUCE_ADD"); // TMA spoofed src tensor
|
||||
|
||||
traits.copy_unpack_(cute::raw_pointer_cast(src.data()), dst.data().coord_, tuple_seq<decltype(dst.data().coord_)>{});
|
||||
}
|
||||
@@ -383,6 +439,13 @@ struct Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, OpArgs...>
|
||||
// 0: uint64_t* bulk_load_memory_barrier
|
||||
cute::tuple<OpArgs...> bulk_load_mbar_;
|
||||
|
||||
// Record the memory barrier for the instruction
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, uint64_t*>
|
||||
with(uint64_t& bulk_mbar) const {
|
||||
return {{&bulk_mbar}};
|
||||
}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr
|
||||
@@ -395,15 +458,29 @@ struct Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, OpArgs...>
|
||||
"Extra arguments not set. Set .with() before use.");
|
||||
static_assert(is_gmem<TS>::value, "Expected gmem src for SM90_BULK_COPY_G2S");
|
||||
static_assert(is_smem<TD>::value, "Expected smem dst for SM90_BULK_COPY_G2S");
|
||||
SM90_BULK_COPY_G2S::copy(raw_pointer_cast(src.data()), *get<0>(traits.bulk_load_mbar_),
|
||||
SM90_BULK_COPY_G2S::copy(raw_pointer_cast(src.data()), get<0>(traits.bulk_load_mbar_),
|
||||
raw_pointer_cast(dst.data()), int32_t(NumBitsPerTMA::value / 8));
|
||||
}
|
||||
};
|
||||
|
||||
// Record the memory barrier for the instruction
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA, uint64_t*>
|
||||
with(uint64_t& bulk_mbar) const {
|
||||
return {{&bulk_mbar}};
|
||||
template <class NumBitsPerTMA, class... Args>
|
||||
struct Copy_Traits<SM90_BULK_COPY_G2S::PREFETCH, NumBitsPerTMA, Args...>
|
||||
: Copy_Traits<SM90_BULK_COPY_G2S, NumBitsPerTMA>
|
||||
{
|
||||
template <class... CopyArgs>
|
||||
CUTE_HOST_DEVICE
|
||||
Copy_Traits(Copy_Traits<CopyArgs...> const& traits) {}
|
||||
|
||||
template <class TS, class SLayout,
|
||||
class TD, class DLayout>
|
||||
CUTE_HOST_DEVICE friend constexpr
|
||||
void
|
||||
copy_unpack(Copy_Traits const& traits,
|
||||
Tensor<TS,SLayout> const& src,
|
||||
Tensor<TD,DLayout> & dst)
|
||||
{
|
||||
static_assert(is_gmem<TS>::value, "Expected gmem src for SM90_BULK_PREFETCH");
|
||||
SM90_BULK_COPY_G2S::PREFETCH::copy(raw_pointer_cast(src.data()), int32_t(NumBitsPerTMA::value / 8));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -653,7 +730,7 @@ template <class GEngine, class GLayout,
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
fill_tma_gmem_shape_stride(Tensor<GEngine,GLayout> const& gtensor, // Gmem Shapes and Strides, in units of TmaInternalType
|
||||
TmaGmemBasisStride const& tma_gbasis_stride, // Map Tma mode idx -> Gmem mode(s)
|
||||
TmaGmemBasisStride const& tma_gbasis_stride, // Map Tma mode idx -> Gmem mode(s)
|
||||
cute::array<ShapeT, TmaRank> & gmem_prob_shape, // Tma Shapes, uint32_t or uin64_t
|
||||
cute::array<uint64_t, TmaRank> & gmem_prob_stride) // Tma Strides
|
||||
{
|
||||
@@ -663,7 +740,7 @@ fill_tma_gmem_shape_stride(Tensor<GEngine,GLayout> const& gtensor, /
|
||||
using TmaInternalType = typename GEngine::value_type;
|
||||
constexpr int tma_rank = decltype(rank(tma_gbasis_stride))::value;
|
||||
static_assert(TmaRank >= tma_rank);
|
||||
|
||||
|
||||
auto gmem_shape = shape(gtensor);
|
||||
auto gmem_stride = stride(gtensor);
|
||||
// Use the indirections in tma_gbasis_stride into gtensor to construct the tma gmem shapes/strides
|
||||
@@ -703,12 +780,12 @@ template <class GEngine, class GLayout,
|
||||
class ShapeT, size_t TmaRank>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
fill_tma_gmem_shape_stride(Copy_Traits<Op,Bits,Aux> const& tma_traits,
|
||||
fill_tma_gmem_shape_stride(Copy_Traits<Op,Bits,Aux> const& tma_traits,
|
||||
Tensor<GEngine,GLayout> const& gtensor, // Gmem Shapes and Strides, value_type = TmaInternalType
|
||||
cute::array<ShapeT, TmaRank> & gmem_prob_shape, // Tma Shapes, uint32_t or uin64_t
|
||||
cute::array<uint64_t, TmaRank> & gmem_prob_stride) // Tma Strides
|
||||
{
|
||||
return fill_tma_gmem_shape_stride(gtensor, stride(typename Aux::TmaGmemBasis{}),
|
||||
return fill_tma_gmem_shape_stride(gtensor, stride(typename Aux::TmaGmemBasis{}),
|
||||
gmem_prob_shape, gmem_prob_stride);
|
||||
}
|
||||
|
||||
@@ -824,7 +901,7 @@ make_tma_copy_desc(Tensor<GEngine,GLayout> const& gtensor, // The origin
|
||||
// Construct the descriptor
|
||||
//
|
||||
|
||||
TmaDescriptor tma_desc = {0};
|
||||
TmaDescriptor tma_desc{};
|
||||
|
||||
//
|
||||
// TMA general info
|
||||
@@ -897,7 +974,7 @@ make_tma_copy_desc(Tensor<GEngine,GLayout> const& gtensor, // The origin
|
||||
if constexpr (decltype(rank<j>(tma_gmem_basis_stride) == Int<1>{})::value) {
|
||||
return E<j>{}; // Return TMA Coord basis -- known scale of Int<1>{}
|
||||
} else {
|
||||
int32_t scale = ceil_div(int32_t(di * sizeof_bits_v<TmaInternalType> / cute::max(gmem_prob_stride[j], 16)), 8);
|
||||
int32_t scale = ceil_div(int32_t(di * sizeof_bits_v<TmaInternalType> / cute::max(gmem_prob_stride[j], uint64_t{16})), 8);
|
||||
return E<j>{} * scale; // Return TMA Coord basis -- with a dynamic scale factor
|
||||
}
|
||||
}
|
||||
@@ -948,7 +1025,7 @@ make_tma_copy_atom(CopyOp,
|
||||
// Construct the Copy_Traits
|
||||
//
|
||||
|
||||
constexpr int num_bits_per_tma = size(tma_gbasis) * sizeof_bits<TmaInternalType>::value;
|
||||
constexpr int num_bits_per_tma = size(tma_gbasis) * sizeof_bits_v<TmaInternalType>;
|
||||
using Traits = Copy_Traits<CopyOp, cute::C<num_bits_per_tma>, decltype(aux_params)>;
|
||||
using Atom = Copy_Atom<Traits, typename GEngine::value_type>;
|
||||
|
||||
@@ -1105,6 +1182,14 @@ make_tma_copy(CopyOp const& copy_op,
|
||||
CTA_Tiler const& cta_tiler,
|
||||
Cluster_Size const& cluster_size)
|
||||
{
|
||||
if constexpr (cute::is_same_v<CopyOp, SM90_TMA_LOAD_IM2COL> ||
|
||||
cute::is_same_v<CopyOp, SM90_TMA_STORE_IM2COL>) {
|
||||
return make_im2col_tma_copy(copy_op,
|
||||
gtensor,
|
||||
slayout,
|
||||
cta_tiler,
|
||||
cluster_size);
|
||||
} else {
|
||||
auto cta_v_tile = make_identity_layout(shape(gtensor)).compose(cta_tiler);
|
||||
auto cta_t_tile = make_layout(cluster_size);
|
||||
// Prefer TmaInternalType if specified. Fallback to GEngine::value_type
|
||||
@@ -1112,6 +1197,7 @@ make_tma_copy(CopyOp const& copy_op,
|
||||
return detail::make_tma_copy_tiled<TmaType>(copy_op,
|
||||
gtensor, slayout,
|
||||
cta_t_tile, cta_v_tile);
|
||||
}
|
||||
}
|
||||
|
||||
// Explicit defaulting
|
||||
@@ -1179,9 +1265,11 @@ auto
|
||||
tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
CtaCoord const& cta_coord,
|
||||
Layout<TShape,TStride> const& cta_layout, // T: CTA coord -> logical multicast id
|
||||
Tensor<SEngine,SLayout> const& stensor, // SMEM Tensor (TMATile, Iter)
|
||||
Tensor<GEngine,GLayout> const& gtensor) // GMEM Tensor (TMATile, Iter)
|
||||
Tensor<SEngine,SLayout> const& stensor, // SMEM Tensor (TMATile, Rest...)
|
||||
Tensor<GEngine,GLayout> const& gtensor) // GMEM Tensor (TMATile, Rest...)
|
||||
{
|
||||
CUTE_STATIC_ASSERT_V(size<0>(stensor) == size<0>(gtensor));
|
||||
|
||||
// Invert the smem to get the largest contiguous vector in the smem layout
|
||||
Layout inv_smem_layout = right_inverse(get_nonswizzle_portion(layout<0>(stensor)));
|
||||
// Scale that up to cover all of the smem_coords
|
||||
@@ -1189,14 +1277,19 @@ tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
|
||||
// Factor out the single-instrucion portion
|
||||
Layout tma_layout_v = make_layout(Int<Copy_Atom<Args...>::NumValSrc>{});
|
||||
Layout layout_V = logical_divide(layout_v, tma_layout_v);
|
||||
|
||||
auto layout_V = make_tile(logical_divide(layout_v, tma_layout_v));
|
||||
|
||||
// Append with _ until we cover all Rest... modes
|
||||
auto glayout_V = append<rank_v<decltype(gtensor)>>(layout_V, _);
|
||||
auto slayout_V = append<rank_v<decltype(stensor)>>(layout_V, _);
|
||||
// Transform tile mode and coalesce
|
||||
Tensor gtensor_v = coalesce(gtensor.compose(layout_V, _), Shape<Shape<_1,_1>,_1>{}); // ((TMA,TMA_Iter),Iter)
|
||||
Tensor stensor_v = coalesce(stensor.compose(layout_V, _), Shape<Shape<_1,_1>,_1>{}); // ((TMA,TMA_Iter),Iter)
|
||||
Tensor gtensor_v = coalesce(gtensor.compose(glayout_V), Shape<Shape<_1,_1>>{}); // ((TMA,TMA_Iter), Rest...)
|
||||
Tensor stensor_v = coalesce(stensor.compose(slayout_V), Shape<Shape<_1,_1>>{}); // ((TMA,TMA_Iter), Rest...)
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print("gtensor : "); print(gtensor); print("\n");
|
||||
print("stensor : "); print(stensor); print("\n");
|
||||
print("layout_V : "); print(layout_V); print("\n");
|
||||
print("gtensor_v : "); print(gtensor_v); print("\n");
|
||||
print("stensor_v : "); print(stensor_v); print("\n");
|
||||
@@ -1205,11 +1298,15 @@ tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
|
||||
// Restride the cta-into-tma-instr layout
|
||||
Layout tma_layout_t = composition(make_layout(Int<1>{}, shape_div(size(tma_layout_v), cosize(cta_layout))), cta_layout);
|
||||
Layout tma_layout_tv = make_layout(tma_layout_t, tma_layout_v);
|
||||
auto tma_layout_tv = make_tile(make_tile(make_layout(tma_layout_t, tma_layout_v), _));
|
||||
|
||||
// Append with _ until we cover all Rest... modes
|
||||
auto gtma_layout_tv = append<rank_v<decltype(gtensor)>>(tma_layout_tv, _);
|
||||
auto stma_layout_tv = append<rank_v<decltype(stensor)>>(tma_layout_tv, _);
|
||||
|
||||
// Transform TMA mode
|
||||
Tensor gtensor_tv = gtensor_v.compose(make_tile(tma_layout_tv, _), _); // (((Thr,Frg),TMA_Iter),Iter)
|
||||
Tensor stensor_tv = stensor_v.compose(make_tile(tma_layout_tv, _), _); // (((Thr,Frg),TMA_Iter),Iter)
|
||||
Tensor gtensor_tv = gtensor_v.compose(gtma_layout_tv); // (((Thr,Frg),TMA_Iter), Rest...)
|
||||
Tensor stensor_tv = stensor_v.compose(stma_layout_tv); // (((Thr,Frg),TMA_Iter), Rest...)
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
@@ -1219,9 +1316,11 @@ tma_partition(Copy_Atom<Args...> const& copy_atom,
|
||||
}
|
||||
#endif
|
||||
|
||||
// Slice and group Frg,TMA_Iter and return
|
||||
auto c = make_coord(make_coord(make_coord(cta_coord, _), _), _);
|
||||
return cute::make_tuple(group_modes<0,2>(gtensor_tv(c)), group_modes<0,2>(stensor_tv(c)));
|
||||
auto c = make_coord(make_coord(make_coord(cta_coord, _), _));
|
||||
auto c_s = append<rank_v<decltype(stensor_tv)>>(c, _);
|
||||
auto c_g = append<rank_v<decltype(gtensor_tv)>>(c, _);
|
||||
|
||||
return cute::make_tuple(group_modes<0,2>(gtensor_tv(c_g)), group_modes<0,2>(stensor_tv(c_s)));
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
#include <cute/arch/mma.hpp>
|
||||
|
||||
#include <cute/atom/mma_traits.hpp>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
|
||||
@@ -78,7 +77,7 @@ struct MMA_Atom<MMA_Traits<Args...>>
|
||||
CUTE_HOST_DEVICE
|
||||
auto
|
||||
with(TraitsArgs&&... args) const {
|
||||
auto traits = Traits::with(std::forward<TraitsArgs>(args)...);
|
||||
auto traits = Traits::with(static_cast<TraitsArgs&&>(args)...);
|
||||
return MMA_Atom<decltype(traits)>{traits};
|
||||
}
|
||||
|
||||
@@ -157,7 +156,7 @@ struct MMA_Atom<MMA_Traits<Args...>>
|
||||
// If the intended FrgTypeA is a view (of the current tensor), forward the whole
|
||||
static_assert(is_same<ValTypeA, typename remove_cvref_t<ATensor>::value_type>::value
|
||||
, "Expecting ValTypeA type");
|
||||
return make_tensor<FrgTypeA>(std::forward<ATensor>(atensor));
|
||||
return make_tensor<FrgTypeA>(static_cast<ATensor&&>(atensor));
|
||||
} else {
|
||||
// Else, the intended FrgTypeA is a value type, construct a new tensor with a fragment layout
|
||||
return make_fragment_like<FrgTypeA>(atensor);
|
||||
@@ -179,7 +178,7 @@ struct MMA_Atom<MMA_Traits<Args...>>
|
||||
// If the intended FrgTypeB is a view (of the current tensor), forward the whole
|
||||
static_assert(is_same<ValTypeB, typename remove_cvref_t<BTensor>::value_type>::value
|
||||
, "Expecting ValTypeB type");
|
||||
return make_tensor<FrgTypeB>(std::forward<BTensor>(btensor));
|
||||
return make_tensor<FrgTypeB>(static_cast<BTensor&&>(btensor));
|
||||
} else {
|
||||
// Else, the intended FrgTypeB is a value type, construct a new tensor with a fragment layout
|
||||
return make_fragment_like<FrgTypeB>(btensor);
|
||||
@@ -213,7 +212,7 @@ struct TiledMMA : MMA_Atom
|
||||
|
||||
static_assert( rank_v<AtomLayoutMNK> == 3, "TiledMMA requires rank-3 AtomLayoutMNK");
|
||||
static_assert( rank_v<PermutationMNK> == 3, "TiledMMA requires rank-3 PermutationMNK");
|
||||
static_assert( is_tile<PermutationMNK>::value, "TiledMMA requires independent permutations of MNK.");
|
||||
static_assert( is_tuple<PermutationMNK>::value, "TiledMMA requires independent permutations of MNK.");
|
||||
static_assert(is_static<PermutationMNK>::value, "TiledMMA requires static permutations of MNK.");
|
||||
|
||||
using ThrLayoutVMNK = decltype(tiled_product(AtomThrID{}, AtomLayoutMNK{}));
|
||||
@@ -391,7 +390,7 @@ struct TiledMMA : MMA_Atom
|
||||
} else {
|
||||
return cute::max(core_size, perm_size);
|
||||
}
|
||||
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
@@ -517,7 +516,7 @@ struct ThrMMA : TiledMMA
|
||||
auto
|
||||
partition_C(CTensor&& ctensor) const
|
||||
{
|
||||
auto thr_tensor = make_tensor(std::forward<CTensor>(ctensor).data(), this->thrfrg_C(ctensor.layout()));
|
||||
auto thr_tensor = make_tensor(static_cast<CTensor&&>(ctensor).data(), this->thrfrg_C(ctensor.layout()));
|
||||
|
||||
auto thr_vmn = make_coord(get<0>(thr_vmnk_), make_coord(get<1>(thr_vmnk_), get<2>(thr_vmnk_)));
|
||||
return thr_tensor(thr_vmn, make_coord(_, repeat<rank<1,1>(thr_tensor)>(_)));
|
||||
@@ -528,7 +527,7 @@ struct ThrMMA : TiledMMA
|
||||
auto
|
||||
partition_A(ATensor&& atensor) const
|
||||
{
|
||||
auto thr_tensor = make_tensor(std::forward<ATensor>(atensor).data(), this->thrfrg_A(atensor.layout()));
|
||||
auto thr_tensor = make_tensor(static_cast<ATensor&&>(atensor).data(), this->thrfrg_A(atensor.layout()));
|
||||
|
||||
auto thr_vmk = make_coord(get<0>(thr_vmnk_), make_coord(get<1>(thr_vmnk_), get<3>(thr_vmnk_)));
|
||||
return thr_tensor(thr_vmk, make_coord(_, repeat<rank<1,1>(thr_tensor)>(_)));
|
||||
@@ -539,7 +538,7 @@ struct ThrMMA : TiledMMA
|
||||
auto
|
||||
partition_B(BTensor&& btensor) const
|
||||
{
|
||||
auto thr_tensor = make_tensor(std::forward<BTensor>(btensor).data(), this->thrfrg_B(btensor.layout()));
|
||||
auto thr_tensor = make_tensor(static_cast<BTensor&&>(btensor).data(), this->thrfrg_B(btensor.layout()));
|
||||
|
||||
auto thr_vnk = make_coord(get<0>(thr_vmnk_), make_coord(get<2>(thr_vmnk_), get<3>(thr_vmnk_)));
|
||||
return thr_tensor(thr_vnk, make_coord(_, repeat<rank<1,1>(thr_tensor)>(_)));
|
||||
@@ -744,7 +743,15 @@ print(ThrMMA<TiledMMA, ThrVMNK> const& thr_mma)
|
||||
|
||||
template <class... Args>
|
||||
CUTE_HOST_DEVICE
|
||||
auto
|
||||
void
|
||||
print_latex(MMA_Atom<Args...> const& mma_atom)
|
||||
{
|
||||
print_latex(make_tiled_mma(mma_atom));
|
||||
}
|
||||
|
||||
template <class... Args>
|
||||
CUTE_HOST_DEVICE
|
||||
void
|
||||
print_latex(TiledMMA<Args...> const& mma)
|
||||
{
|
||||
auto layout_and_thrid_C = mma.get_layoutC_MN();
|
||||
@@ -764,7 +771,7 @@ print_latex(TiledMMA<Args...> const& mma)
|
||||
layoutB_NK, thrID_B);
|
||||
}
|
||||
|
||||
// MNK MMA Layout to console printer -- 8-value color coded by thread
|
||||
// MNK MMA Layout to console printer
|
||||
template <class LayoutC, class ThrIDC,
|
||||
class LayoutA, class ThrIDA,
|
||||
class LayoutB, class ThrIDB>
|
||||
|
||||
@@ -32,12 +32,8 @@
|
||||
|
||||
#include <cute/arch/mma_sm80.hpp>
|
||||
#include <cute/atom/mma_traits.hpp>
|
||||
|
||||
#include <cute/layout.hpp>
|
||||
|
||||
#include <cute/numeric/integer_subbyte.hpp>
|
||||
|
||||
#include <cutlass/numeric_types.h>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
+15
-19
@@ -98,16 +98,16 @@
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#if defined(_MSC_VER)
|
||||
// Provides support for alternative operators 'and', 'or', and 'not'
|
||||
#include <iso646.h>
|
||||
# include <iso646.h>
|
||||
#endif // _MSC_VER
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#define CUTE_STL_NAMESPACE cuda::std
|
||||
#define CUTE_STL_NAMESPACE_IS_CUDA_STD
|
||||
# define CUTE_STL_NAMESPACE cuda::std
|
||||
# define CUTE_STL_NAMESPACE_IS_CUDA_STD
|
||||
#else
|
||||
#define CUTE_STL_NAMESPACE std
|
||||
# define CUTE_STL_NAMESPACE std
|
||||
#endif
|
||||
|
||||
//
|
||||
@@ -115,9 +115,9 @@
|
||||
//
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cassert>
|
||||
# include <cuda/std/cassert>
|
||||
#else
|
||||
#include <cassert>
|
||||
# include <cassert>
|
||||
#endif
|
||||
|
||||
#define CUTE_STATIC_V(x) decltype(x)::value
|
||||
@@ -125,10 +125,11 @@
|
||||
#define CUTE_STATIC_ASSERT static_assert
|
||||
#define CUTE_STATIC_ASSERT_V(x,...) static_assert(decltype(x)::value, ##__VA_ARGS__)
|
||||
|
||||
// Fail and print a message. Typically used for notification of a compiler misconfiguration.
|
||||
#if defined(__CUDA_ARCH__)
|
||||
# define CUTE_RUNTIME_ASSERT(x) __brkpt()
|
||||
# define CUTE_INVALID_CONTROL_PATH(x) assert(0 && x); printf(x); __brkpt()
|
||||
#else
|
||||
# define CUTE_RUNTIME_ASSERT(x) assert(0 && x)
|
||||
# define CUTE_INVALID_CONTROL_PATH(x) assert(0 && x); printf(x)
|
||||
#endif
|
||||
|
||||
//
|
||||
@@ -136,9 +137,9 @@
|
||||
//
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
# include <cstdio>
|
||||
# include <iostream>
|
||||
# include <iomanip>
|
||||
#endif
|
||||
|
||||
//
|
||||
@@ -151,13 +152,8 @@
|
||||
// Basic types
|
||||
//
|
||||
|
||||
#include <cute/numeric/int.hpp>
|
||||
#include <cute/numeric/real.hpp>
|
||||
#include <cute/numeric/half.hpp>
|
||||
#include <cute/numeric/float8.hpp>
|
||||
#include <cute/numeric/bfloat.hpp>
|
||||
#include <cute/numeric/tfloat.hpp>
|
||||
#include <cute/numeric/complex.hpp>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
//
|
||||
// Debugging utilities
|
||||
//
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/numeric/int.hpp>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
#include <cute/numeric/math.hpp>
|
||||
|
||||
namespace cute
|
||||
|
||||
@@ -355,7 +355,7 @@ void clear(array<T,N>& a)
|
||||
a.fill(T(0));
|
||||
}
|
||||
|
||||
template <typename T, size_t N>
|
||||
template <class T, size_t N>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void fill(array<T,N>& a, T const& value)
|
||||
{
|
||||
@@ -370,14 +370,14 @@ void swap(array<T,N>& a, array<T,N>& b)
|
||||
}
|
||||
|
||||
/// @return A cute::array of the elements of @c t in reverse order.
|
||||
template <typename T, size_t N>
|
||||
CUTE_HOST_DEVICE constexpr cute::array<T, N>
|
||||
reverse(cute::array<T, N> const& t) {
|
||||
template <class T, size_t N>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
cute::array<T,N> reverse(cute::array<T,N> const& t)
|
||||
{
|
||||
if constexpr (N == 0u) {
|
||||
return t;
|
||||
}
|
||||
else {
|
||||
cute::array<T, N> t_r{};
|
||||
} else {
|
||||
cute::array<T,N> t_r{};
|
||||
for (size_t k = 0; k < N; ++k) {
|
||||
t_r[k] = t[N - k - 1];
|
||||
}
|
||||
@@ -422,7 +422,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
T&& get(array<T,N>&& a)
|
||||
{
|
||||
static_assert(I < N, "Index out of range");
|
||||
return std::move(a[I]);
|
||||
return cute::move(a[I]);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
@@ -442,12 +442,12 @@ struct tuple_element<I, cute::array<T,N>>
|
||||
};
|
||||
|
||||
template <class T, size_t N>
|
||||
struct tuple_size<const cute::array<T,N>>
|
||||
struct tuple_size<cute::array<T,N> const>
|
||||
: CUTE_STL_NAMESPACE::integral_constant<size_t, N>
|
||||
{};
|
||||
|
||||
template <size_t I, class T, size_t N>
|
||||
struct tuple_element<I, const cute::array<T,N>>
|
||||
struct tuple_element<I, cute::array<T,N> const>
|
||||
{
|
||||
using type = T;
|
||||
};
|
||||
@@ -462,7 +462,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
@@ -478,12 +478,12 @@ struct tuple_element<I, cute::array<T,N>>
|
||||
};
|
||||
|
||||
template <class T, size_t N>
|
||||
struct tuple_size<const cute::array<T,N>>
|
||||
struct tuple_size<cute::array<T,N> const>
|
||||
: CUTE_STL_NAMESPACE::integral_constant<size_t, N>
|
||||
{};
|
||||
|
||||
template <size_t I, class T, size_t N>
|
||||
struct tuple_element<I, const cute::array<T,N>>
|
||||
struct tuple_element<I, cute::array<T,N> const>
|
||||
{
|
||||
using type = T;
|
||||
};
|
||||
|
||||
@@ -37,29 +37,20 @@
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/numeric/int.hpp> // sizeof_bits
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
#include <cute/numeric/integral_constant.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
template <class T>
|
||||
struct is_subbyte {
|
||||
static constexpr bool value = sizeof_bits_v<T> < 8;
|
||||
};
|
||||
|
||||
template <class T>
|
||||
constexpr bool is_subbyte_v = is_subbyte<T>::value;
|
||||
|
||||
//
|
||||
// Underlying subbyte storage type
|
||||
//
|
||||
template <class T>
|
||||
using subbyte_storage_type_t = conditional_t<(sizeof_bits_v<T> <= 8), uint8_t,
|
||||
conditional_t<(sizeof_bits_v<T> <= 16), uint16_t,
|
||||
conditional_t<(sizeof_bits_v<T> <= 32), uint32_t,
|
||||
conditional_t<(sizeof_bits_v<T> <= 64), uint64_t,
|
||||
conditional_t<(sizeof_bits_v<T> <= 128), uint128_t,
|
||||
using subbyte_storage_type_t = conditional_t<(cute::sizeof_bits_v<T> <= 8), uint8_t,
|
||||
conditional_t<(cute::sizeof_bits_v<T> <= 16), uint16_t,
|
||||
conditional_t<(cute::sizeof_bits_v<T> <= 32), uint32_t,
|
||||
conditional_t<(cute::sizeof_bits_v<T> <= 64), uint64_t,
|
||||
conditional_t<(cute::sizeof_bits_v<T> <= 128), uint128_t,
|
||||
T>>>>>;
|
||||
|
||||
template <class T> struct subbyte_iterator;
|
||||
@@ -183,6 +174,11 @@ public:
|
||||
operator element_type() const {
|
||||
return get();
|
||||
}
|
||||
|
||||
// Address
|
||||
subbyte_iterator<T> operator&() const {
|
||||
return {ptr_, idx_};
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
@@ -314,7 +310,7 @@ public:
|
||||
CUTE_HOST_DEVICE constexpr friend
|
||||
auto recast_ptr(subbyte_iterator const& x) {
|
||||
using NewT = conditional_t<(is_const_v<T>), NewT_ const, NewT_>;
|
||||
if constexpr (is_subbyte<NewT>::value) { // Making subbyte_iter, preserve the subbyte idx
|
||||
if constexpr (cute::is_subbyte_v<NewT>) { // Making subbyte_iter, preserve the subbyte idx
|
||||
return subbyte_iterator<NewT>(x.ptr_, x.idx_);
|
||||
} else { // Not subbyte, assume/assert subbyte idx 0
|
||||
return reinterpret_cast<NewT*>(raw_pointer_cast(x));
|
||||
@@ -323,7 +319,7 @@ public:
|
||||
}
|
||||
|
||||
CUTE_HOST_DEVICE friend void print(subbyte_iterator x) {
|
||||
printf("subptr[%db](%p.%u)", int(sizeof_bits<T>::value), x.ptr_, x.idx_);
|
||||
printf("subptr[%db](%p.%u)", int(sizeof_bits_v<T>), x.ptr_, x.idx_);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -369,8 +365,8 @@ private:
|
||||
|
||||
public:
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
array_subbyte() {}
|
||||
constexpr
|
||||
array_subbyte() = default;
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
array_subbyte(array_subbyte const& x) {
|
||||
@@ -562,7 +558,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
T&& get(array_subbyte<T,N>&& a)
|
||||
{
|
||||
static_assert(I < N, "Index out of range");
|
||||
return std::move(a[I]);
|
||||
return cute::move(a[I]);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
@@ -608,7 +604,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/numeric/int.hpp> // uint_bit_t
|
||||
#include <cute/numeric/numeric_types.hpp> // uint_bit_t
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
@@ -96,11 +96,11 @@ uint32_t&& get(dim3&& a)
|
||||
{
|
||||
static_assert(I < 3, "Index out of range");
|
||||
if constexpr (I == 0) {
|
||||
return std::move(a.x);
|
||||
return cute::move(a.x);
|
||||
} else if constexpr (I == 1) {
|
||||
return std::move(a.y);
|
||||
return cute::move(a.y);
|
||||
} else if constexpr (I == 2) {
|
||||
return std::move(a.z);
|
||||
return cute::move(a.z);
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
@@ -162,11 +162,11 @@ uint32_t&& get(uint3&& a)
|
||||
{
|
||||
static_assert(I < 3, "Index out of range");
|
||||
if constexpr (I == 0) {
|
||||
return std::move(a.x);
|
||||
return cute::move(a.x);
|
||||
} else if constexpr (I == 1) {
|
||||
return std::move(a.y);
|
||||
return cute::move(a.y);
|
||||
} else if constexpr (I == 2) {
|
||||
return std::move(a.z);
|
||||
return cute::move(a.z);
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
|
||||
@@ -126,18 +126,14 @@ CUTE_HOST_DEVICE constexpr T& getv(EBO<N, T, false>& x)
|
||||
|
||||
template <size_t N, class T>
|
||||
CUTE_HOST_DEVICE constexpr T&& getv(EBO<N, T, false>&& x)
|
||||
{ return static_cast<T&&>(x.t_); }
|
||||
{ return cute::move(x.t_); }
|
||||
|
||||
template <class IdxSeq, class... T>
|
||||
struct TupleBase;
|
||||
|
||||
// Base class of cute::tuple.
|
||||
// It inherits from EBO<i, t> for each (i, t) in (I..., T...).
|
||||
// The actual storage (for nonempty t) lives in the base classes.
|
||||
// index_sequence is a way to wrap up a sequence of zero or more
|
||||
// compile-time integer values in a single type.
|
||||
// We only ever use index_sequence<0, 1, ..., sizeof...(T)> in practice,
|
||||
// as the type alias TupleBase below indicates.
|
||||
// Base class of cute::tuple binds each element to an index
|
||||
// by inheriting from EBO<i, t> for each (i, t) in (I..., T...).
|
||||
// The storage (for nonempty t) lives in the base classes.
|
||||
template <size_t... I, class... T>
|
||||
struct TupleBase<index_sequence<I...>, T...>
|
||||
: EBO<I,T>...
|
||||
@@ -169,11 +165,6 @@ struct TupleBase<index_sequence<I...>, T...>
|
||||
//
|
||||
// Inheriting from the above alias TupleBase
|
||||
// causes MSVC 2022 build errors when assigning one tuple to another:
|
||||
//
|
||||
// illegal member initialization:
|
||||
// 'TupleBase< /* template arguments */ >' is not a base or member
|
||||
//
|
||||
// Not using the alias or any kind of alias fixed the errors.
|
||||
// In summary: this is verbose as a work-around for MSVC build errors.
|
||||
template <class... T>
|
||||
struct tuple : detail::TupleBase<make_index_sequence<sizeof...(T)>, T...>
|
||||
@@ -365,10 +356,10 @@ tuple_cat(T0 const& t0, T1 const& t1, T2 const& t2, T3 const& t3, T4 const& t4,
|
||||
return cute::make_tuple(get<I0>(t0)..., get<I1>(t1)..., get<I2>(t2)..., get<I3>(t3)..., get<I4>(t4)...);
|
||||
}
|
||||
|
||||
template<class T0, class T1>
|
||||
template <class T0, class T1>
|
||||
struct tuple_cat_static;
|
||||
|
||||
template<class... T0s, class... T1s>
|
||||
template <class... T0s, class... T1s>
|
||||
struct tuple_cat_static<tuple<T0s...>, tuple<T1s...>> {
|
||||
using type = tuple<T0s..., T1s...>;
|
||||
};
|
||||
@@ -630,11 +621,8 @@ template <class Tuple, size_t... Is>
|
||||
CUTE_HOST_DEVICE void print_tuple(Tuple const& t,
|
||||
index_sequence<Is...>, char s = '(', char e = ')')
|
||||
{
|
||||
using eat = int[];
|
||||
using cute::print;
|
||||
(void) eat {(print(s), 0),
|
||||
(print(Is == 0 ? "" : ","), print(get<Is>(t)), 0)...,
|
||||
(print(e), 0)};
|
||||
((void(print(Is == 0 ? s : ',')), void(print(get<Is>(t)))), ...); print(e);
|
||||
}
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
@@ -642,11 +630,8 @@ template <class Tuple, std::size_t... Is>
|
||||
CUTE_HOST std::ostream& print_tuple_os(std::ostream& os, Tuple const& t,
|
||||
index_sequence<Is...>, char s = '(', char e = ')')
|
||||
{
|
||||
using eat = int[];
|
||||
(void) eat {(void(os << s), 0),
|
||||
(void(os << (Is == 0 ? "" : ",") << get<Is>(t)), 0)...,
|
||||
(void(os << e), 0)};
|
||||
return os;
|
||||
(void(os << (Is == 0 ? s : ',') << get<Is>(t)), ...);
|
||||
return os << e;
|
||||
}
|
||||
#endif // !defined(__CUDACC_RTC__)
|
||||
|
||||
@@ -707,7 +692,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
|
||||
+19
-34
@@ -218,7 +218,7 @@ static constexpr int depth_v = depth_t<Tuple>::value;
|
||||
// product
|
||||
//
|
||||
|
||||
// Implementation of product (see below) as a function object
|
||||
// Implementation of product as a function object
|
||||
struct Product
|
||||
{
|
||||
template <class IntTuple>
|
||||
@@ -232,7 +232,7 @@ struct Product
|
||||
} else {
|
||||
return cute::transform_apply(a, Product{}, multiplies_unary_lfold{});
|
||||
}
|
||||
} else {
|
||||
} else if constexpr (cute::is_integral<IntTuple>::value) {
|
||||
return a;
|
||||
}
|
||||
|
||||
@@ -248,7 +248,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
product_each(Tuple const& t)
|
||||
{
|
||||
return transform(wrap(t), [](auto const& x) { return product(x); });
|
||||
return transform(wrap(t), product);
|
||||
}
|
||||
|
||||
// Take the product of Tuple at the leaves of TupleG
|
||||
@@ -394,7 +394,7 @@ shape_div(IntTupleA const& a, IntTupleB const& b)
|
||||
static_assert(IntTupleA::value % IntTupleB::value == 0 || IntTupleB::value % IntTupleA::value == 0, "Static shape_div failure");
|
||||
return C<shape_div(IntTupleA::value, IntTupleB::value)>{};
|
||||
} else { // int int
|
||||
//assert(a % b == 0 || b % a == 0); // Wave dynamic assertion
|
||||
//assert(a % b == 0 || b % a == 0); // Waive dynamic assertion
|
||||
return a / b != 0 ? a / b : signum(a) * signum(b); // Division with rounding away from zero
|
||||
}
|
||||
|
||||
@@ -855,7 +855,10 @@ elem_geq(T const& t, U const& u) {
|
||||
return !elem_less(t, u);
|
||||
}
|
||||
|
||||
namespace detail {
|
||||
|
||||
/** Increment a (dynamic) coord lexicographically within a shape
|
||||
* @pre is_congruent<Coord,Shape>::value
|
||||
* \code
|
||||
* auto shape = make_shape(1,2,make_shape(2,3),3);
|
||||
*
|
||||
@@ -866,44 +869,26 @@ elem_geq(T const& t, U const& u) {
|
||||
* assert(i == size(shape));
|
||||
* \endcode
|
||||
*/
|
||||
template <class Coord, class Shape>
|
||||
template <int I = 0, class Coord, class Shape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
increment(Coord& coord, Shape const& shape);
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <class Coord, class Shape, int I0, int... Is>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
increment(Coord& coord, Shape const& shape, seq<I0,Is...>)
|
||||
increment(Coord& coord, Shape const& shape)
|
||||
{
|
||||
cute::increment(get<I0>(coord), get<I0>(shape));
|
||||
if constexpr (sizeof...(Is) != 0) {
|
||||
if (back(get<I0>(coord)) == back(get<I0>(shape))) {
|
||||
back(get<I0>(coord)) = 0;
|
||||
increment(coord, shape, seq<Is...>{});
|
||||
if constexpr (is_integral<Coord>::value) {
|
||||
++coord;
|
||||
} else {
|
||||
increment(get<I>(coord), get<I>(shape));
|
||||
if constexpr (I+1 < tuple_size<Coord>::value) {
|
||||
if (back(get<I>(coord)) == back(get<I>(shape))) {
|
||||
back(get<I>(coord)) = 0;
|
||||
increment<I+1>(coord, shape);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
template <class Coord, class Shape>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
void
|
||||
increment(Coord& coord, Shape const& shape)
|
||||
{
|
||||
if constexpr (is_integral<Coord>::value && is_integral<Shape>::value) {
|
||||
++coord;
|
||||
} else if constexpr (is_tuple<Coord>::value && is_tuple<Shape>::value) {
|
||||
static_assert(tuple_size<Coord>::value == tuple_size<Shape>::value, "Mismatched ranks");
|
||||
detail::increment(coord, shape, tuple_seq<Coord>{});
|
||||
} else {
|
||||
static_assert(sizeof(Coord) == 0, "Invalid parameters");
|
||||
}
|
||||
}
|
||||
|
||||
struct ForwardCoordIteratorSentinal
|
||||
{};
|
||||
|
||||
@@ -918,7 +903,7 @@ struct ForwardCoordIterator
|
||||
Coord const& operator*() const { return coord; }
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ForwardCoordIterator& operator++() { increment(coord, shape); return *this; }
|
||||
ForwardCoordIterator& operator++() { detail::increment(coord, shape); return *this; }
|
||||
|
||||
// Sentinel for the end of the implied range
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
|
||||
+67
-41
@@ -56,6 +56,9 @@ using Step = cute::tuple<Strides...>;
|
||||
template <class... Coords>
|
||||
using Coord = cute::tuple<Coords...>;
|
||||
|
||||
template <class... Layouts>
|
||||
using Tile = cute::tuple<Layouts...>;
|
||||
|
||||
template <class... Ts>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Shape<Ts...>
|
||||
@@ -80,7 +83,17 @@ Coord<Ts...>
|
||||
make_coord(Ts const&... t) {
|
||||
return {t...};
|
||||
}
|
||||
template <class... Ts>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
Tile<Ts...>
|
||||
make_tile(Ts const&... t)
|
||||
{
|
||||
return {t...};
|
||||
}
|
||||
|
||||
//
|
||||
// Layout
|
||||
//
|
||||
|
||||
template <class Shape, class Stride = LayoutLeft::Apply<Shape> >
|
||||
struct Layout
|
||||
@@ -366,59 +379,56 @@ make_layout(Shape const& shape, GenRowMajor)
|
||||
return make_layout(shape, compact_row_major(shape));
|
||||
}
|
||||
|
||||
// Follow the same ordering induced by the strides, but make the layout compact
|
||||
//
|
||||
// Advanced Layout constructions
|
||||
//
|
||||
|
||||
// Make a compact layout with shape @a shape and strides following the order induced by @a order.
|
||||
// Dynamic values in @a order are ignored, considered large, and considered ordered from left to right.
|
||||
// Example:
|
||||
// make_ordered_layout(Shape<_2,_2,_2,_2>{}, Step<_0,_2,_3,_1>{})
|
||||
// -> (_2,_2,_2,_2):(_1,_4,_8,_2)
|
||||
// make_ordered_layout(make_shape(2,3,4,5), make_step(Int<2>{}, 67, 42, Int<50>{}))
|
||||
// -> (2,3,4,5):(_1,10,30,2)
|
||||
template <class Shape, class Order>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
make_ordered_layout(Shape const& shape, Order const& order)
|
||||
{
|
||||
static_assert(is_static<Order>::value);
|
||||
return make_layout(shape, compact_order(shape, order));
|
||||
}
|
||||
|
||||
template <class Shape, class Stride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
make_ordered_layout(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
return make_ordered_layout(layout.shape(), layout.stride());
|
||||
}
|
||||
|
||||
// Make a layout of the same shape that is either ordered or colmajor depending on staticness
|
||||
// Make a compact layout with the same shape as @a layout
|
||||
// and strides following the order induced by @a layout.stride().
|
||||
// Static-0 strides in the input @a layout are preserved in the output.
|
||||
// Example:
|
||||
// make_layout_like(Layout<Shape<_2,_2,_2,_2>, Stride<_0,_2,_4,_1>>{})
|
||||
// -> (_2,_2,_2,_2):(_0,_2,_4,_1)
|
||||
// make_layout_like(make_layout(make_shape(2,3,4,5), make_stride(Int<0>{},42,Int<1>{},Int<0>{})))
|
||||
// -> (2,3,4,5):(_0,4,_1,_0)
|
||||
template <class Shape, class Stride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
make_layout_like(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
auto any_zero = any_of(layout.stride(), [](auto d) { return is_constant<0, decltype(d)>{}; });
|
||||
if constexpr (any_zero) {
|
||||
// If there are static-0 strides, then make a col-major layout that keeps those 0s
|
||||
return make_layout(layout.shape(),
|
||||
compact_col_major(filter_zeros(layout.stride(), layout.shape())));
|
||||
} else
|
||||
if constexpr (is_static<Shape>::value && is_static<Stride>::value) {
|
||||
// If the layout is fully static, then make a layout that follows the same order as the strides
|
||||
// Assumes the strides are unique
|
||||
return make_ordered_layout(layout.shape(), layout.stride());
|
||||
} else {
|
||||
return make_layout(layout.shape());
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
return make_layout(layout.shape(),
|
||||
compact_order(filter_zeros(layout.stride(), layout.shape()), layout.stride()));
|
||||
}
|
||||
|
||||
//
|
||||
// Make a layout of the same shape,
|
||||
// with mode-0 being colmajor then following the mode order in layout
|
||||
//
|
||||
// Make a compact layout with the same shape as @a layout
|
||||
// and strides following the order induced by @a layout.stride(),
|
||||
// except mode-0 is always stride-1 and generated column-major.
|
||||
// The 0th mode is commonly used for MMA_Atoms or Copy_Atoms
|
||||
// so this generates the 0th mode with LayoutLeft regardless of the reference layout.
|
||||
template <class Shape, class Stride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
make_fragment_like(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
constexpr int R = Layout<Shape,Stride>::rank;
|
||||
if constexpr (R > 1 && is_static<Shape>::value && is_static<Stride>::value) {
|
||||
return tiled_product(make_layout(shape<0>(layout)), make_ordered_layout(take<1,R>(layout)));
|
||||
if constexpr (R > 1 && is_static<Shape>::value) {
|
||||
return tiled_product(make_layout(shape<0>(layout)),
|
||||
make_ordered_layout(take<1,R>(layout.shape()), take<1,R>(layout.stride())));
|
||||
} else {
|
||||
return make_layout(layout.shape());
|
||||
}
|
||||
@@ -458,11 +468,11 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
get(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
return make_layout(get<Is...>(layout.shape()),
|
||||
return make_layout(get<Is...>(layout.shape()),
|
||||
get<Is...>(layout.stride()));
|
||||
}
|
||||
|
||||
// Return a new layout with only the modes in the range [B,E)
|
||||
// Return a new layout with only the modes in the range [B,E)
|
||||
template <int B, int E, class Shape, class Stride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -470,7 +480,7 @@ take(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
static_assert(B < E, "take: empty range error");
|
||||
static_assert(0 <= B && E <= Layout<Shape,Stride>::rank, "take: range out of bounds");
|
||||
return make_layout(take<B,E>(layout.shape()),
|
||||
return make_layout(take<B,E>(layout.shape()),
|
||||
take<B,E>(layout.stride()));
|
||||
}
|
||||
|
||||
@@ -490,7 +500,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
flatten(Layout<Shape,Stride> const& layout)
|
||||
{
|
||||
return make_layout(flatten(layout.shape()),
|
||||
return make_layout(flatten(layout.shape()),
|
||||
flatten(layout.stride()));
|
||||
}
|
||||
|
||||
@@ -1376,6 +1386,23 @@ logical_divide(Layout<LShape,LStride> const& layout,
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
// Generalization of ceil_div for Layout lhs
|
||||
// is effectively the "rest mode" of logical_divide.
|
||||
// Occurs in the calculation of gridDim, for example, for generalized tilers
|
||||
// Example:
|
||||
// dim3 gridDim(size(ceil_div(problem_shape_M, cta_tiler_M)),
|
||||
// size(ceil_div(problem_shape_N, cta_tiler_N)));
|
||||
// This does not consider compositional acceptance, so it may be the case that
|
||||
// ceil_div produces a result while logical_divide (and friends) do not.
|
||||
template <class Target, class TShape, class TStride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
ceil_div(Target const& target,
|
||||
Layout<TShape,TStride> const& tiler)
|
||||
{
|
||||
return complement(tiler, size(target));
|
||||
}
|
||||
|
||||
//
|
||||
// Convenience operator
|
||||
// that produces layouts like ((BLK_A,BLK_B,...),(a,b,...,x,y))
|
||||
@@ -1425,7 +1452,6 @@ flat_divide(Layout<LShape,LStride> const& layout,
|
||||
// Logical product
|
||||
//
|
||||
|
||||
// @post compatible()
|
||||
template <class LShape, class LStride,
|
||||
class TShape, class TStride>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
@@ -1501,7 +1527,7 @@ flat_product(Layout<LShape,LStride> const& block,
|
||||
|
||||
//
|
||||
// Rank-sensitive products
|
||||
//
|
||||
//
|
||||
|
||||
// blocked_product -- Reproduce a block over a tiler.
|
||||
// Think of every element of "tiler" as a "block"
|
||||
@@ -1517,7 +1543,7 @@ blocked_product(Layout<TShape,TStride> const& block,
|
||||
constexpr int R = cute::max(rank_v<TShape>, rank_v<UShape>);
|
||||
|
||||
auto result = logical_product(append<R>(block), append<R>(tiler));
|
||||
|
||||
|
||||
return coalesce(zip(get<0>(result), get<1>(result)), tuple_repeat<R>(Int<1>{}));
|
||||
}
|
||||
|
||||
@@ -1545,7 +1571,7 @@ raked_product(Layout<TShape,TStride> const& block,
|
||||
// @param block The layout to repeat
|
||||
// @param trg_shape The target shape of the result
|
||||
// @param ord_shape The order of the modes of @a trg_shape to tile @a layout with.
|
||||
// Defaults to GenColMajor, so @a layout will repeat
|
||||
// Defaults to GenColMajor, so @a layout will repeat
|
||||
// across the first mode first, the second mode second, etc
|
||||
// E.g. Step<_2,_1,_3> will cause @a layout to repeat
|
||||
// across the second mode first, the first mode second, and the third mode last.
|
||||
@@ -1659,7 +1685,7 @@ recast_layout(Layout<Shape,Stride> const& layout)
|
||||
else if constexpr (scale::num == 1) {
|
||||
return downcast<scale::den>(layout);
|
||||
}
|
||||
else if constexpr (scale::den == 1) {
|
||||
else if constexpr (scale::den == 1) {
|
||||
return upcast<scale::num>(layout);
|
||||
}
|
||||
else {
|
||||
|
||||
@@ -178,6 +178,16 @@ struct ComposedLayout : private cute::tuple<LayoutA, Offset, LayoutB> // EBO fo
|
||||
tile(Layouts const&... layouts) const {
|
||||
return tiled_divide(*this, make_tile(layouts...));
|
||||
}
|
||||
|
||||
// Equality, return a static or dynamic boolean
|
||||
template <class... Args>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator==(ComposedLayout<Args...> const& other) const {
|
||||
return this->layout_a() == other.layout_a() &&
|
||||
this->layout_b() == other.layout_b() &&
|
||||
this->offset() == other.offset();
|
||||
}
|
||||
};
|
||||
|
||||
template <class A, class O, class B>
|
||||
|
||||
@@ -63,6 +63,9 @@ struct ArithmeticTuple : tuple<T...>
|
||||
template <class... T>
|
||||
struct is_tuple<ArithmeticTuple<T...>> : true_type {};
|
||||
|
||||
template <class... Ts>
|
||||
struct is_flat<ArithmeticTuple<Ts...>> : is_flat<tuple<Ts...>> {};
|
||||
|
||||
template <class... T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
@@ -108,16 +111,45 @@ template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator+(ArithmeticTuple<T...> const& t, tuple<U...> const& u) {
|
||||
constexpr int R = cute::max(int(sizeof...(T)), int(sizeof...(U)));
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), plus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
return t + ArithmeticTuple<U...>(u);
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator+(tuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
return ArithmeticTuple<T...>(t) + u;
|
||||
}
|
||||
|
||||
// Subtraction
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
constexpr int R = cute::max(int(sizeof...(T)), int(sizeof...(U)));
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), plus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
return transform_apply(append<R>(t,Int<0>{}), append<R>(u,Int<0>{}), minus{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t, tuple<U...> const& u) {
|
||||
return t - ArithmeticTuple<U...>(u);
|
||||
}
|
||||
|
||||
template <class... T, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(tuple<T...> const& t, ArithmeticTuple<U...> const& u) {
|
||||
return ArithmeticTuple<T...>(t) - u;
|
||||
}
|
||||
|
||||
// Negation
|
||||
template <class... T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator-(ArithmeticTuple<T...> const& t) {
|
||||
return transform_apply(t, negate{}, [](auto const&... a){ return make_arithmetic_tuple(a...); });
|
||||
}
|
||||
|
||||
//
|
||||
@@ -128,7 +160,7 @@ template <auto t, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<U...> const&
|
||||
operator+(C<t>, ArithmeticTuple<U...> const& u) {
|
||||
static_assert(t == 0, "Artihmetic tuple op+ error!");
|
||||
static_assert(t == 0, "Arithmetic tuple op+ error!");
|
||||
return u;
|
||||
}
|
||||
|
||||
@@ -136,7 +168,23 @@ template <class... T, auto u>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<T...> const&
|
||||
operator+(ArithmeticTuple<T...> const& t, C<u>) {
|
||||
static_assert(u == 0, "Artihmetic tuple op+ error!");
|
||||
static_assert(u == 0, "Arithmetic tuple op+ error!");
|
||||
return t;
|
||||
}
|
||||
|
||||
template <auto t, class... U>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<U...> const&
|
||||
operator-(C<t>, ArithmeticTuple<U...> const& u) {
|
||||
static_assert(t == 0, "Arithmetic tuple op- error!");
|
||||
return -u;
|
||||
}
|
||||
|
||||
template <class... T, auto u>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
ArithmeticTuple<T...> const&
|
||||
operator-(ArithmeticTuple<T...> const& t, C<u>) {
|
||||
static_assert(u == 0, "Arithmetic tuple op- error!");
|
||||
return t;
|
||||
}
|
||||
|
||||
@@ -531,7 +579,7 @@ namespace std
|
||||
template <class... _Tp>
|
||||
struct tuple_size;
|
||||
|
||||
template<size_t _Ip, class... _Tp>
|
||||
template <size_t _Ip, class... _Tp>
|
||||
struct tuple_element;
|
||||
#endif
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::bfloat16_t;
|
||||
|
||||
//
|
||||
// Display utilities
|
||||
//
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
CUTE_HOST std::ostream& operator<<(std::ostream& os, bfloat16_t const& v)
|
||||
{
|
||||
return os << float(v);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // end namespace cute
|
||||
@@ -30,9 +30,9 @@
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cutlass/complex.h>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cute/numeric/numeric_types.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -44,6 +44,9 @@ using cutlass::real;
|
||||
using cutlass::imag;
|
||||
using cutlass::conj;
|
||||
|
||||
template <class T>
|
||||
static constexpr auto is_complex_v = is_complex<T>::value;
|
||||
|
||||
/// Fused multiply-add for complex numbers
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::half_t;
|
||||
|
||||
} // end namespace cute
|
||||
@@ -36,8 +36,7 @@
|
||||
#include <cstdint>
|
||||
#endif
|
||||
|
||||
#include <cute/numeric/integer_subbyte.hpp>
|
||||
#include <cute/numeric/uint128.hpp>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -46,16 +45,16 @@ namespace cute
|
||||
// Signed integers
|
||||
//
|
||||
|
||||
using int2_t = cute::int2b_t;
|
||||
using int4_t = cute::int4b_t;
|
||||
using int8_t = CUTE_STL_NAMESPACE::int8_t;
|
||||
using int16_t = CUTE_STL_NAMESPACE::int16_t;
|
||||
using int32_t = CUTE_STL_NAMESPACE::int32_t;
|
||||
using int64_t = CUTE_STL_NAMESPACE::int64_t;
|
||||
using int2_t = cutlass::int2b_t;
|
||||
using int4_t = cutlass::int4b_t;
|
||||
using CUTE_STL_NAMESPACE::int8_t;
|
||||
using CUTE_STL_NAMESPACE::int16_t;
|
||||
using CUTE_STL_NAMESPACE::int32_t;
|
||||
using CUTE_STL_NAMESPACE::int64_t;
|
||||
|
||||
template <int N> struct int_bit;
|
||||
template <> struct int_bit< 2> { using type = cute::int2b_t; };
|
||||
template <> struct int_bit< 4> { using type = cute::int4b_t; };
|
||||
template <> struct int_bit< 2> { using type = cutlass::int2b_t; };
|
||||
template <> struct int_bit< 4> { using type = cutlass::int4b_t; };
|
||||
template <> struct int_bit< 8> { using type = int8_t; };
|
||||
template <> struct int_bit< 16> { using type = int16_t; };
|
||||
template <> struct int_bit< 32> { using type = int32_t; };
|
||||
@@ -74,24 +73,24 @@ using int_byte_t = typename int_byte<N>::type;
|
||||
// Unsigned integers
|
||||
//
|
||||
|
||||
using uint1_t = cute::uint1b_t;
|
||||
using uint2_t = cute::uint2b_t;
|
||||
using uint4_t = cute::uint4b_t;
|
||||
using uint8_t = CUTE_STL_NAMESPACE::uint8_t;
|
||||
using uint16_t = CUTE_STL_NAMESPACE::uint16_t;
|
||||
using uint32_t = CUTE_STL_NAMESPACE::uint32_t;
|
||||
using uint64_t = CUTE_STL_NAMESPACE::uint64_t;
|
||||
using uint128_t = cute::uint128_t;
|
||||
using uint1_t = cutlass::uint1b_t;
|
||||
using uint2_t = cutlass::uint2b_t;
|
||||
using uint4_t = cutlass::uint4b_t;
|
||||
using CUTE_STL_NAMESPACE::uint8_t;
|
||||
using CUTE_STL_NAMESPACE::uint16_t;
|
||||
using CUTE_STL_NAMESPACE::uint32_t;
|
||||
using CUTE_STL_NAMESPACE::uint64_t;
|
||||
using cutlass::uint128_t;
|
||||
|
||||
template <int N> struct uint_bit;
|
||||
template <> struct uint_bit< 1> { using type = cute::uint1b_t; };
|
||||
template <> struct uint_bit< 2> { using type = cute::uint2b_t; };
|
||||
template <> struct uint_bit< 4> { using type = cute::uint4b_t; };
|
||||
template <> struct uint_bit< 1> { using type = cutlass::uint1b_t; };
|
||||
template <> struct uint_bit< 2> { using type = cutlass::uint2b_t; };
|
||||
template <> struct uint_bit< 4> { using type = cutlass::uint4b_t; };
|
||||
template <> struct uint_bit< 8> { using type = uint8_t; };
|
||||
template <> struct uint_bit< 16> { using type = uint16_t; };
|
||||
template <> struct uint_bit< 32> { using type = uint32_t; };
|
||||
template <> struct uint_bit< 64> { using type = uint64_t; };
|
||||
template <> struct uint_bit<128> { using type = cute::uint128_t; };
|
||||
template <> struct uint_bit<128> { using type = cutlass::uint128_t; };
|
||||
|
||||
template <int N>
|
||||
using uint_bit_t = typename uint_bit<N>::type;
|
||||
@@ -102,50 +101,4 @@ using uint_byte = uint_bit<8*N>;
|
||||
template <int N>
|
||||
using uint_byte_t = typename uint_byte<N>::type;
|
||||
|
||||
//
|
||||
// sizeof_bytes
|
||||
//
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bytes {
|
||||
static constexpr size_t value = sizeof(T);
|
||||
};
|
||||
template <class T>
|
||||
static constexpr int sizeof_bytes_v = sizeof_bytes<T>::value;
|
||||
|
||||
//
|
||||
// sizeof_bits
|
||||
//
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bits {
|
||||
static constexpr size_t value = sizeof(T) * 8;
|
||||
};
|
||||
|
||||
template <class T>
|
||||
struct sizeof_bits<T const>: sizeof_bits<T> {};
|
||||
|
||||
template <>
|
||||
struct sizeof_bits<void> {
|
||||
static constexpr size_t value = 0;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct sizeof_bits<bool> {
|
||||
static constexpr size_t value = 1;
|
||||
};
|
||||
|
||||
template <int Bits, bool Signed>
|
||||
struct sizeof_bits<integer_subbyte<Bits,Signed>> {
|
||||
static constexpr size_t value = Bits;
|
||||
};
|
||||
|
||||
template <int Bits, bool Signed>
|
||||
struct sizeof_bits<cutlass::integer_subbyte<Bits,Signed>> {
|
||||
static constexpr size_t value = Bits;
|
||||
};
|
||||
|
||||
template <class T>
|
||||
static constexpr int sizeof_bits_v = sizeof_bits<T>::value;
|
||||
|
||||
} // namespace cute
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cstdint>
|
||||
#else
|
||||
#include <cstdint>
|
||||
#endif
|
||||
|
||||
#include <cutlass/integer_subbyte.h>
|
||||
|
||||
#include <cute/config.hpp>
|
||||
#include <cute/util/type_traits.hpp>
|
||||
|
||||
namespace cute {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <int Bits, bool Signed = true>
|
||||
struct integer_subbyte
|
||||
{
|
||||
/// Storage type
|
||||
using Storage = uint8_t;
|
||||
|
||||
/// Number of bits
|
||||
static_assert(Bits <= 8*sizeof(Storage), "Require a subbyte of bits in integer_subbyte");
|
||||
|
||||
/// External type
|
||||
using xint_t = typename conditional<Signed, int, unsigned>::type;
|
||||
|
||||
/// Bitmask for truncation from larger integers
|
||||
static constexpr Storage bits_mask_ = Storage((1 << Bits) - 1);
|
||||
/// Bitmask for the sign bit
|
||||
static constexpr Storage sign_mask_ = Storage((Signed ? 1 : 0) << (Bits - 1));
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
Storage storage;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// No operation
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte() {}
|
||||
|
||||
/// Conversion from integer type
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte(int value) // NOTE: Sign extension?
|
||||
: storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
|
||||
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
integer_subbyte(unsigned value)
|
||||
: storage(reinterpret_cast<Storage const&>(value) & bits_mask_) {}
|
||||
|
||||
/// Convert to int or unsigned
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
operator xint_t() const {
|
||||
if (sign_mask_ & storage) { // Sign extend
|
||||
return xint_t(storage) | ~xint_t(bits_mask_);
|
||||
} else {
|
||||
return xint_t(storage);
|
||||
}
|
||||
}
|
||||
|
||||
/// Equality
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator==(integer_subbyte const& rhs) const {
|
||||
return storage == rhs.storage;
|
||||
}
|
||||
|
||||
/// Inequality
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator!=(integer_subbyte const& rhs) const {
|
||||
return storage != rhs.storage;
|
||||
}
|
||||
|
||||
/// Less than or equal
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator<=(integer_subbyte const& rhs) const {
|
||||
if (sign_mask_ & storage) {
|
||||
return !(rhs.storage < storage);
|
||||
} else {
|
||||
return storage <= rhs.storage;
|
||||
}
|
||||
}
|
||||
|
||||
/// Less than
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator<(integer_subbyte const& rhs) const {
|
||||
if (sign_mask_ & storage) {
|
||||
return !(rhs.storage <= storage);
|
||||
} else {
|
||||
return storage < rhs.storage;
|
||||
}
|
||||
}
|
||||
|
||||
/// Greater than or equal
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator>=(integer_subbyte const& rhs) const {
|
||||
return !(*this < rhs);
|
||||
}
|
||||
|
||||
/// Greater than
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
bool operator>(integer_subbyte const& rhs) const {
|
||||
return !(*this <= rhs);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// 1-bit unsigned integer type
|
||||
using uint1b_t = integer_subbyte<1, false>;
|
||||
|
||||
/// 2-bit integer type
|
||||
using int2b_t = integer_subbyte<2, true>;
|
||||
|
||||
/// 2-bit unsigned integer type
|
||||
using uint2b_t = integer_subbyte<2, false>;
|
||||
|
||||
/// 4-bit integer type
|
||||
using int4b_t = integer_subbyte<4, true>;
|
||||
|
||||
/// 4-bit unsigned integer type
|
||||
using uint4b_t = integer_subbyte<4, false>;
|
||||
|
||||
/// 1-bit binary type
|
||||
using bin1_t = bool;
|
||||
|
||||
} // namespace cute
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
|
||||
#include <limits>
|
||||
|
||||
namespace CUTE_STL_NAMESPACE {
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint1b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint1b_t const max() noexcept { return 1; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::int2b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t lowest() noexcept { return -2; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t min() noexcept { return -2; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int2b_t max() noexcept { return 1; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = true;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint2b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint2b_t const max() noexcept { return 3; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::int4b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t lowest() noexcept { return -8; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t min() noexcept { return -8; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::int4b_t max() noexcept { return 7; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = true;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct numeric_limits<cute::uint4b_t> {
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const lowest() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const min() noexcept { return 0; }
|
||||
CUTE_HOST_DEVICE static constexpr
|
||||
cute::uint4b_t const max() noexcept { return 15; }
|
||||
static constexpr bool is_integer = true;
|
||||
static constexpr bool is_signed = false;
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
|
||||
#endif // !defined(__CUDACC_RTC__)
|
||||
@@ -443,4 +443,35 @@ CUTE_HOST std::ostream& operator<<(std::ostream& os, C<t> const&) {
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
namespace detail {
|
||||
|
||||
// parse_int_digits takes a variadic number of digits and converts them into an int
|
||||
template <class... Ts>
|
||||
constexpr uint64_t parse_int_digits(uint64_t result, int digit, Ts... digits)
|
||||
{
|
||||
if constexpr (sizeof...(Ts) == 0) {
|
||||
return 10 * result + digit;
|
||||
} else {
|
||||
return parse_int_digits(10 * result + digit, digits...);
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace detail
|
||||
|
||||
|
||||
// This user-defined literal operator allows cute::constant written as literals. For example,
|
||||
//
|
||||
// auto var = 32_c;
|
||||
//
|
||||
// var has type cute::constant<int,32>.
|
||||
//
|
||||
template <char... digits>
|
||||
constexpr cute::constant<int,detail::parse_int_digits(0, (digits - '0')...)> operator "" _c()
|
||||
{
|
||||
static_assert((('0' <= digits && digits <= '9') && ...),
|
||||
"Expected 0 <= digit <= 9 for each digit of the integer.");
|
||||
return {};
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
|
||||
@@ -130,6 +130,10 @@ nratio(R<a,b>, R<c,d>) {
|
||||
return {};
|
||||
}
|
||||
|
||||
//
|
||||
// Operators
|
||||
//
|
||||
|
||||
template <auto a, auto b, auto x, auto y>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
typename R<a*x,b*y>::type
|
||||
@@ -227,14 +231,14 @@ abs(R<a,b>) {
|
||||
|
||||
template <auto a, auto b>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
int32_t
|
||||
log_2(R<a,b>) {
|
||||
static_assert(R<a,b>::num > 0);
|
||||
static_assert(R<a,b>::den > 0);
|
||||
return log_2(static_cast<uint32_t>(R<a,b>::num)) - log_2(static_cast<uint32_t>(R<a,b>::den));
|
||||
}
|
||||
|
||||
|
||||
// @return A non-reduced ratio cute::R of the Trait0::value / Trait1::value
|
||||
template <class Trait0, class Trait1>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
|
||||
@@ -316,11 +316,11 @@ safe_div(T const& t, U const& u) {
|
||||
|
||||
template <class T>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
int32_t
|
||||
log_2(T x) {
|
||||
assert(x > 0);
|
||||
static_assert(is_unsigned<T>::value, "Only to be used for unsigned integral types.");
|
||||
return bit_width(x) - 1;
|
||||
return static_cast<int32_t>(bit_width(x)) - 1;
|
||||
}
|
||||
|
||||
} // namespace cute
|
||||
|
||||
@@ -30,14 +30,46 @@
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
#include <cutlass/numeric_size.h>
|
||||
|
||||
#include <cute/numeric/int.hpp>
|
||||
#include <cute/numeric/real.hpp>
|
||||
|
||||
namespace cute {
|
||||
|
||||
template <typename T>
|
||||
struct sizeof_bits : public cutlass::sizeof_bits<T> {};
|
||||
|
||||
// DO NOT change auto to int, sizeof_bits<sparse_elem> use integral_ratio instead of int
|
||||
template <class T>
|
||||
static constexpr auto sizeof_bits_v = sizeof_bits<T>::value;
|
||||
|
||||
using cutlass::bits_to_bytes;
|
||||
|
||||
using cutlass::is_subbyte;
|
||||
|
||||
template <class T>
|
||||
static constexpr auto is_subbyte_v = is_subbyte<T>::value;
|
||||
|
||||
using cutlass::half_t;
|
||||
using cutlass::bfloat16_t;
|
||||
|
||||
using cutlass::tfloat32_t;
|
||||
|
||||
// Umbrella floating-point 8-bit data type : type_erased_dynamic_float8_t
|
||||
// This umbrella datatype can be enabled when a user provides a specific
|
||||
// datatype in runtime argument list.
|
||||
using cutlass::type_erased_dynamic_float8_t;
|
||||
using cutlass::float_e4m3_t;
|
||||
using cutlass::float_e5m2_t;
|
||||
|
||||
using cutlass::uint1b_t;
|
||||
using cutlass::int2b_t;
|
||||
using cutlass::uint2b_t;
|
||||
using cutlass::int4b_t;
|
||||
using cutlass::uint4b_t;
|
||||
using cutlass::bin1_t;
|
||||
|
||||
} // end namespace cute
|
||||
@@ -1,53 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <vector_types.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
|
||||
namespace cute {
|
||||
|
||||
using cutlass::tfloat32_t;
|
||||
|
||||
//
|
||||
// Display utilities
|
||||
//
|
||||
|
||||
#if !defined(__CUDACC_RTC__)
|
||||
CUTE_HOST std::ostream& operator<<(std::ostream& os, tfloat32_t const& v)
|
||||
{
|
||||
return os << float(v);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // end namespace cute
|
||||
@@ -1,259 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#if defined(__CUDACC_RTC__)
|
||||
#include <cuda/std/cstdint>
|
||||
#else
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
#include <type_traits>
|
||||
#include <stdexcept>
|
||||
#endif
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
/// Optionally enable GCC's built-in type
|
||||
#if defined(__x86_64) && !defined(__CUDA_ARCH__)
|
||||
# if defined(__GNUC__) && 0
|
||||
# define CUTE_UINT128_NATIVE
|
||||
# elif defined(_MSC_VER)
|
||||
# define CUTE_INT128_ARITHMETIC
|
||||
# include <intrin.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cute {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
///! Unsigned 128b integer type
|
||||
struct alignas(16) uint128_t
|
||||
{
|
||||
/// Size of one part of the uint's storage in bits
|
||||
static constexpr int storage_bits_ = 64;
|
||||
|
||||
struct hilo
|
||||
{
|
||||
uint64_t lo;
|
||||
uint64_t hi;
|
||||
};
|
||||
|
||||
// Use a union to store either low and high parts or, if present, a built-in 128b integer type.
|
||||
union
|
||||
{
|
||||
struct hilo hilo_;
|
||||
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
unsigned __int128 native;
|
||||
#endif // defined(CUTE_UINT128_NATIVE)
|
||||
};
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Default ctor
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t() : hilo_{0, 0} {}
|
||||
|
||||
/// Constructor from uint64
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t(uint64_t lo_) : hilo_{lo_, 0} {}
|
||||
|
||||
/// Constructor from two 64b unsigned integers
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t(uint64_t lo_, uint64_t hi_) : hilo_{lo_, hi_} {}
|
||||
|
||||
/// Optional constructor from native value
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
uint128_t(unsigned __int128 value) : native(value) { }
|
||||
#endif
|
||||
|
||||
/// Lossily cast to uint64
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
explicit operator uint64_t() const
|
||||
{
|
||||
return hilo_.lo;
|
||||
}
|
||||
|
||||
template <class Dummy = bool>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
static void exception()
|
||||
{
|
||||
//static_assert(sizeof(Dummy) == 0, "Not implemented exception!");
|
||||
//abort();
|
||||
//printf("uint128 not implemented!\n");
|
||||
}
|
||||
|
||||
/// Add
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator+(uint128_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native + rhs.native;
|
||||
#else
|
||||
y.hilo_.lo = hilo_.lo + rhs.hilo_.lo;
|
||||
y.hilo_.hi = hilo_.hi + rhs.hilo_.hi + (!y.hilo_.lo && (rhs.hilo_.lo));
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Subtract
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator-(uint128_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native - rhs.native;
|
||||
#else
|
||||
y.hilo_.lo = hilo_.lo - rhs.hilo_.lo;
|
||||
y.hilo_.hi = hilo_.hi - rhs.hilo_.hi - (rhs.hilo_.lo && y.hilo_.lo > hilo_.lo);
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Multiply by unsigned 64b integer yielding 128b integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator*(uint64_t const& rhs) const
|
||||
{
|
||||
uint128_t y;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
y.native = native * rhs;
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// Multiply by the low part
|
||||
y.hilo_.lo = _umul128(hilo_.lo, rhs, &y.hilo_.hi);
|
||||
|
||||
// Add the high part and ignore the overflow
|
||||
uint64_t overflow;
|
||||
y.hilo_.hi += _umul128(hilo_.hi, rhs, &overflow);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return y;
|
||||
}
|
||||
|
||||
/// Divide 128b operation by 64b operation yielding a 64b quotient
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t operator/(uint64_t const& divisor) const
|
||||
{
|
||||
uint64_t quotient = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
quotient = uint64_t(native / divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
uint64_t remainder = 0;
|
||||
quotient = _udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return quotient;
|
||||
}
|
||||
|
||||
/// Divide 128b operation by 64b operation yielding a 64b quotient
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t operator%(uint64_t const& divisor) const
|
||||
{
|
||||
uint64_t remainder = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
remainder = uint64_t(native % divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
(void)_udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return remainder;
|
||||
}
|
||||
|
||||
/// Computes the quotient and remainder in a single method.
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint64_t divmod(uint64_t &remainder, uint64_t divisor) const
|
||||
{
|
||||
uint64_t quotient = 0;
|
||||
#if defined(CUTE_UINT128_NATIVE)
|
||||
quotient = uint64_t(native / divisor);
|
||||
remainder = uint64_t(native % divisor);
|
||||
#elif defined(CUTE_INT128_ARITHMETIC)
|
||||
// implemented using MSVC's arithmetic intrinsics
|
||||
quotient = _udiv128(hilo_.hi, hilo_.lo, divisor, &remainder);
|
||||
#else
|
||||
exception();
|
||||
#endif
|
||||
return quotient;
|
||||
}
|
||||
|
||||
/// Left-shifts a 128b unsigned integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator<<(int sh) const
|
||||
{
|
||||
if (sh == 0) {
|
||||
return *this;
|
||||
}
|
||||
else if (sh >= storage_bits_) {
|
||||
return uint128_t(0, hilo_.lo << (sh - storage_bits_));
|
||||
}
|
||||
else {
|
||||
return uint128_t(
|
||||
(hilo_.lo << sh),
|
||||
(hilo_.hi << sh) | uint64_t(hilo_.lo >> (storage_bits_ - sh))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Right-shifts a 128b unsigned integer
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
uint128_t operator>>(int sh) const
|
||||
{
|
||||
if (sh == 0) {
|
||||
return *this;
|
||||
}
|
||||
else if (sh >= storage_bits_) {
|
||||
return uint128_t((hilo_.hi >> (sh - storage_bits_)), 0);
|
||||
}
|
||||
else {
|
||||
return uint128_t(
|
||||
(hilo_.lo >> sh) | (hilo_.hi << (storage_bits_ - sh)),
|
||||
(hilo_.hi >> sh)
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace cute
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -33,7 +33,7 @@
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cute/numeric/int.hpp> // sizeof_bits
|
||||
#include <cute/numeric/numeric_types.hpp> // sizeof_bits
|
||||
#include <cute/numeric/math.hpp>
|
||||
#include <cute/numeric/integral_constant.hpp>
|
||||
|
||||
@@ -58,7 +58,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
recast_ptr(void* ptr)
|
||||
{
|
||||
if constexpr (is_subbyte<NewT>::value) {
|
||||
if constexpr (cute::is_subbyte_v<NewT>) {
|
||||
return subbyte_iterator<NewT>(ptr);
|
||||
} else {
|
||||
return reinterpret_cast<NewT*>(ptr);
|
||||
@@ -71,7 +71,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
recast_ptr(void const* ptr)
|
||||
{
|
||||
if constexpr (is_subbyte<NewT>::value) {
|
||||
if constexpr (cute::is_subbyte_v<NewT>) {
|
||||
return subbyte_iterator<NewT const>(ptr);
|
||||
} else {
|
||||
return reinterpret_cast<NewT const*>(ptr);
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/util/type_traits.hpp>
|
||||
#include <cute/numeric/int.hpp> // sizeof_bits
|
||||
#include <cute/numeric/numeric_types.hpp> // sizeof_bits
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
@@ -109,7 +109,7 @@ as_position_independent_swizzle_tensor(Tensor&& tensor)
|
||||
} else {
|
||||
#if !defined(NDEBUG)
|
||||
{
|
||||
uint32_t address = cast_smem_ptr_to_uint(raw_pointer_cast(std::forward<Tensor>(tensor).data()));
|
||||
uint32_t address = cast_smem_ptr_to_uint(raw_pointer_cast(static_cast<Tensor&&>(tensor).data()));
|
||||
uint32_t mask = ((uint32_t(1) << SwizzleFn::num_base) - 1) | SwizzleFn::swizzle_code;
|
||||
assert((address & mask) == 0); // Alignment to the Base, Z, and Y of Swizzle
|
||||
}
|
||||
@@ -118,7 +118,7 @@ as_position_independent_swizzle_tensor(Tensor&& tensor)
|
||||
// Recast swizzle from acting on byte-addressed pointers to elements of type-T
|
||||
auto new_swizzle = recast_layout<uint8_t, T>(SwizzleFn{});
|
||||
// Strip off everything and create a new smem_ptr for type-T
|
||||
auto new_ptr = make_smem_ptr<T>(raw_pointer_cast(std::forward<Tensor>(tensor).data()));
|
||||
auto new_ptr = make_smem_ptr<T>(raw_pointer_cast(static_cast<Tensor&&>(tensor).data()));
|
||||
return make_tensor(new_ptr, composition(new_swizzle, Int<0>{}, tensor.layout()));
|
||||
}
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
|
||||
+38
-17
@@ -394,20 +394,26 @@ compact_row_major(Shape const& shape,
|
||||
|
||||
namespace detail {
|
||||
|
||||
template <class Shape, class Order, class OrigShape, class OrigOrder>
|
||||
// @pre weakly_congruent(order, shape)
|
||||
// @pre is_congruent<RefShape, RefOrder>
|
||||
// @pre is_static<Order>
|
||||
// @pre is_static<RefOrder>
|
||||
template <class Shape, class Order, class RefShape, class RefOrder>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
compact_order(Shape const& shape, Order const& order,
|
||||
OrigShape const& orig_shape, OrigOrder const& orig_order)
|
||||
RefShape const& ref_shape, RefOrder const& ref_order)
|
||||
{
|
||||
if constexpr (is_tuple<Order>::value) {
|
||||
return transform(shape, order, [&](auto const& x, auto const& y) { return compact_order(x, y, orig_shape, orig_order); });
|
||||
static_assert(tuple_size<Shape>::value == tuple_size<Order>::value, "Need equal rank of shape and order");
|
||||
return transform(shape, order, [&](auto const& s, auto const& o) { return compact_order(s, o, ref_shape, ref_order); });
|
||||
} else {
|
||||
auto d = product(transform(orig_shape, orig_order,
|
||||
[&](auto const& s, auto const& o) {
|
||||
return conditional_return(o < order, product(s), Int<1>{});
|
||||
}));
|
||||
return compact_col_major(shape, d);
|
||||
// Compute the starting stride for this shape by accumulating all shapes corresponding to lesser orders
|
||||
auto stride_start = product(transform(ref_shape, ref_order,
|
||||
[&](auto const& s, auto const& o) {
|
||||
return conditional_return(o < order, s, Int<1>{});
|
||||
}));
|
||||
return compact_col_major(shape, stride_start);
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
@@ -420,15 +426,30 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
compact_order(Shape const& shape, Order const& order)
|
||||
{
|
||||
if constexpr(is_congruent<Shape,Order>::value) {
|
||||
return detail::compact_order(shape, order, flatten_to_tuple(shape), flatten_to_tuple(order));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Here we only want to apply order to top-level subshapes and default (col-major) order on other levels
|
||||
static_assert(rank(Shape{}) == rank(Order{}), "Need equal rank of shape and order");
|
||||
return detail::compact_order(shape, order, shape, order);
|
||||
}
|
||||
auto ref_shape = flatten_to_tuple(product_like(shape, order));
|
||||
|
||||
auto flat_order = flatten_to_tuple(order);
|
||||
// Find the largest static element of order
|
||||
auto max_order = cute::fold(flat_order, Int<0>{}, [](auto v, auto order) {
|
||||
if constexpr (is_constant<true, decltype(v < order)>::value) {
|
||||
return order;
|
||||
} else {
|
||||
return v;
|
||||
}
|
||||
});
|
||||
// Replace any dynamic elements within order with large-static elements
|
||||
auto max_seq = make_range<max_order+1, max_order+1+rank(flat_order)>{};
|
||||
auto ref_order = cute::transform(max_seq, flat_order, [](auto seq_v, auto order) {
|
||||
if constexpr (is_static<decltype(order)>::value) {
|
||||
return order;
|
||||
} else {
|
||||
return seq_v;
|
||||
}
|
||||
});
|
||||
|
||||
auto new_order = unflatten(ref_order, order);
|
||||
|
||||
return detail::compact_order(shape, new_order, ref_shape, ref_order);
|
||||
}
|
||||
|
||||
template <class Shape>
|
||||
|
||||
@@ -87,6 +87,14 @@ struct Swizzle
|
||||
{
|
||||
return apply(offset);
|
||||
}
|
||||
|
||||
template <int B, int M, int S>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
operator==(Swizzle<B,M,S> const&) const
|
||||
{
|
||||
return B == BBits && M == MBase && S == SShift;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
|
||||
+45
-40
@@ -42,7 +42,6 @@
|
||||
|
||||
#include <cute/pointer.hpp>
|
||||
#include <cute/layout.hpp>
|
||||
#include <cute/tile.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
@@ -481,7 +480,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto)
|
||||
tensor(Tensor&& tensor)
|
||||
{
|
||||
return std::forward<Tensor>(tensor);
|
||||
return static_cast<Tensor&&>(tensor);
|
||||
}
|
||||
|
||||
template <int I, int... Is, class Tensor,
|
||||
@@ -490,17 +489,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto)
|
||||
tensor(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), get<I,Is...>(tensor.layout()));
|
||||
}
|
||||
|
||||
// Return the subtensor of a range of modes
|
||||
template <int B, int E, class Tensor,
|
||||
__CUTE_REQUIRES(is_tensor<remove_cvref_t<Tensor>>::value)>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto)
|
||||
take(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), take<B,E>(tensor.layout()));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), get<I,Is...>(tensor.layout()));
|
||||
}
|
||||
|
||||
// Return the layout of a mode
|
||||
@@ -567,7 +556,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
flatten(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), flatten(tensor.layout()));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), flatten(tensor.layout()));
|
||||
}
|
||||
|
||||
template <class Tensor,
|
||||
@@ -576,7 +565,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
coalesce(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), coalesce(tensor.layout()));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), coalesce(tensor.layout()));
|
||||
}
|
||||
|
||||
template <class Tensor, class Profile,
|
||||
@@ -585,7 +574,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
coalesce(Tensor&& tensor, Profile const& profile)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), coalesce(tensor.layout(), profile));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), coalesce(tensor.layout(), profile));
|
||||
}
|
||||
|
||||
template <class Tensor,
|
||||
@@ -594,7 +583,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
filter_zeros(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(cute::forward<Tensor>(tensor).data(), filter_zeros(tensor.layout()));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), filter_zeros(tensor.layout()));
|
||||
}
|
||||
|
||||
template <class Tensor,
|
||||
@@ -603,7 +592,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
filter(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), filter(tensor.layout()));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), filter(tensor.layout()));
|
||||
}
|
||||
|
||||
template <class Tensor, class Profile,
|
||||
@@ -612,7 +601,7 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
filter(Tensor&& tensor, Profile const& profile)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(), filter(tensor.layout(), profile));
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), filter(tensor.layout(), profile));
|
||||
}
|
||||
|
||||
// Return a tensor with the same shape as input but offset by a given coordinate
|
||||
@@ -623,7 +612,7 @@ auto
|
||||
domain_offset(Coord const& coord, Tensor&& tensor)
|
||||
{
|
||||
auto [layout, ptr_offset] = domain_offset(coord, tensor.layout());
|
||||
return make_tensor(std::forward<Tensor>(tensor).data() + ptr_offset, layout);
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data() + ptr_offset, layout);
|
||||
}
|
||||
|
||||
// Group the modes [B,E) into a single mode
|
||||
@@ -635,10 +624,20 @@ CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
group_modes(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(),
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(),
|
||||
group<B,E>(tensor.layout()));
|
||||
}
|
||||
|
||||
// Return the subtensor of a range of modes
|
||||
template <int B, int E, class Tensor,
|
||||
__CUTE_REQUIRES(is_tensor<remove_cvref_t<Tensor>>::value)>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
decltype(auto)
|
||||
take(Tensor&& tensor)
|
||||
{
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(), take<B,E>(tensor.layout()));
|
||||
}
|
||||
|
||||
//
|
||||
// Recast
|
||||
//
|
||||
@@ -662,9 +661,9 @@ recast(Tensor&& tensor)
|
||||
auto extent_diff = transform(shape_diff, flatten(old_layout.stride()), multiplies{});
|
||||
auto offset = fold(extent_diff, Int<0>{}, [](auto const& i, auto const& a) { return i + cute::min(a,Int<0>{}); });
|
||||
|
||||
return make_tensor(recast_ptr<NewType>(std::forward<Tensor>(tensor).data() + offset), new_layout);
|
||||
return make_tensor(recast_ptr<NewType>(static_cast<Tensor&&>(tensor).data() + offset), new_layout);
|
||||
} else {
|
||||
return make_tensor(recast_ptr<NewType>(std::forward<Tensor>(tensor).data() ), new_layout);
|
||||
return make_tensor(recast_ptr<NewType>(static_cast<Tensor&&>(tensor).data() ), new_layout);
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
@@ -788,7 +787,7 @@ auto
|
||||
logical_divide(Tensor && tensor,
|
||||
Tiler const& tiler) // Layout or Tile<Layout...> or Shape
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(),
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(),
|
||||
logical_divide(tensor.layout(), tiler));
|
||||
}
|
||||
|
||||
@@ -802,7 +801,7 @@ auto
|
||||
zipped_divide(Tensor && tensor,
|
||||
Tiler const& tiler) // Layout or Tile<Layout...> or Shape
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(),
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(),
|
||||
zipped_divide(tensor.layout(), tiler));
|
||||
}
|
||||
|
||||
@@ -811,10 +810,10 @@ template <class Tensor, class Tiler,
|
||||
__CUTE_REQUIRES(is_tensor<remove_cvref_t<Tensor>>::value)>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
tiled_divide(Tensor && tensor,
|
||||
tiled_divide(Tensor && tensor,
|
||||
Tiler const& tiler) // Layout or Tile<Layout...> or Shape
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(),
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(),
|
||||
tiled_divide(tensor.layout(), tiler));
|
||||
}
|
||||
|
||||
@@ -826,7 +825,7 @@ auto
|
||||
flat_divide(Tensor && tensor,
|
||||
Tiler const& tiler) // Layout or Tile<Layout...> or Shape
|
||||
{
|
||||
return make_tensor(std::forward<Tensor>(tensor).data(),
|
||||
return make_tensor(static_cast<Tensor&&>(tensor).data(),
|
||||
flat_divide(tensor.layout(), tiler));
|
||||
}
|
||||
|
||||
@@ -850,7 +849,7 @@ inner_partition(Tensor && tensor,
|
||||
Tiler const& tiler,
|
||||
Coord const& coord)
|
||||
{
|
||||
auto tensor_tiled = zipped_divide(std::forward<Tensor>(tensor), tiler);
|
||||
auto tensor_tiled = zipped_divide(static_cast<Tensor&&>(tensor), tiler);
|
||||
constexpr int R0 = decltype(rank<0>(tensor_tiled))::value;
|
||||
|
||||
// The coord slices into the second mode (the "rest" mode), flatten the first
|
||||
@@ -877,7 +876,7 @@ outer_partition(Tensor && tensor,
|
||||
Tiler const& tiler,
|
||||
Coord const& coord)
|
||||
{
|
||||
auto tensor_tiled = zipped_divide(std::forward<Tensor>(tensor), tiler);
|
||||
auto tensor_tiled = zipped_divide(static_cast<Tensor&&>(tensor), tiler);
|
||||
constexpr int R1 = decltype(rank<1>(tensor_tiled))::value;
|
||||
|
||||
// The coord slices into the first mode (the "tile" mode), flatten the second
|
||||
@@ -903,7 +902,7 @@ local_tile(Tensor && tensor,
|
||||
Tiler const& tiler, // tiler to apply
|
||||
Coord const& coord) // coord to slice into "remainder"
|
||||
{
|
||||
return inner_partition(std::forward<Tensor>(tensor),
|
||||
return inner_partition(static_cast<Tensor&&>(tensor),
|
||||
tiler,
|
||||
coord);
|
||||
}
|
||||
@@ -928,7 +927,7 @@ local_tile(Tensor && tensor,
|
||||
Coord const& coord, // coord to slice into "remainder"
|
||||
Proj const& proj) // projection to apply to tiler and coord
|
||||
{
|
||||
return local_tile(std::forward<Tensor>(tensor),
|
||||
return local_tile(static_cast<Tensor&&>(tensor),
|
||||
dice(proj, tiler),
|
||||
dice(proj, coord));
|
||||
}
|
||||
@@ -946,7 +945,7 @@ local_partition(Tensor && tensor,
|
||||
Index const& index) // index to slice for
|
||||
{
|
||||
static_assert(is_integral<Index>::value);
|
||||
return outer_partition(std::forward<Tensor>(tensor),
|
||||
return outer_partition(static_cast<Tensor&&>(tensor),
|
||||
product_each(shape(tile)),
|
||||
tile.get_flat_coord(index));
|
||||
}
|
||||
@@ -970,7 +969,7 @@ local_partition(Tensor && tensor,
|
||||
Index const& index, // index to slice for
|
||||
Projection const& proj)
|
||||
{
|
||||
return local_partition(std::forward<Tensor>(tensor),
|
||||
return local_partition(static_cast<Tensor&&>(tensor),
|
||||
dice(proj, tile),
|
||||
index);
|
||||
}
|
||||
@@ -986,9 +985,11 @@ CUTE_HOST_DEVICE void print(Tensor<Engine,Layout> const& tensor)
|
||||
}
|
||||
|
||||
template <class Engine, class Layout>
|
||||
CUTE_HOST_DEVICE void print_tensor(Tensor<Engine,Layout> const& tensor)
|
||||
CUTE_HOST_DEVICE void print_tensor(Tensor<Engine,Layout> const& tensor, bool print_type = true)
|
||||
{
|
||||
print(tensor); print(":\n");
|
||||
if (print_type) {
|
||||
print(tensor); print(":\n");
|
||||
}
|
||||
|
||||
if constexpr (Layout::rank == 1)
|
||||
{
|
||||
@@ -1008,18 +1009,18 @@ CUTE_HOST_DEVICE void print_tensor(Tensor<Engine,Layout> const& tensor)
|
||||
} else
|
||||
if constexpr (Layout::rank == 3)
|
||||
{
|
||||
print_tensor(tensor(_,_,0));
|
||||
print_tensor(tensor(_,_,0), false);
|
||||
for (int k = 1; k < size<2>(tensor); ++k) {
|
||||
for (int i = 0; i < 5*size<1>(tensor); ++i) { print("-"); } print("\n");
|
||||
print_tensor(tensor(_,_,k));
|
||||
print_tensor(tensor(_,_,k), false);
|
||||
}
|
||||
} else
|
||||
if constexpr (Layout::rank == 4)
|
||||
{
|
||||
print_tensor(tensor(_,_,_,0));
|
||||
print_tensor(tensor(_,_,_,0), false);
|
||||
for (int p = 1; p < size<3>(tensor); ++p) {
|
||||
for (int i = 0; i < 5*size<1>(tensor); ++i) { print("="); } print("\n");
|
||||
print_tensor(tensor(_,_,_,p));
|
||||
print_tensor(tensor(_,_,_,p), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1090,5 +1091,9 @@ CUTE_HOST std::ostream& operator<<(std::ostream& os, Tensor<Engine,Layout> const
|
||||
#include <cute/algorithm/fill.hpp>
|
||||
#include <cute/algorithm/clear.hpp>
|
||||
#include <cute/algorithm/copy.hpp>
|
||||
#include <cute/algorithm/prefetch.hpp>
|
||||
#include <cute/algorithm/axpby.hpp>
|
||||
#include <cute/algorithm/gemm.hpp>
|
||||
|
||||
#include <cute/algorithm/cooperative_copy.hpp>
|
||||
#include <cute/algorithm/cooperative_gemm.hpp>
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#pragma once
|
||||
|
||||
#include <cute/config.hpp>
|
||||
|
||||
#include <cute/layout.hpp>
|
||||
|
||||
namespace cute
|
||||
{
|
||||
|
||||
//
|
||||
// A Tile is not a Layout, it's a tuple of Layouts or Tiles or Underscores
|
||||
//
|
||||
|
||||
template <class... Layouts>
|
||||
using Tile = tuple<Layouts...>;
|
||||
|
||||
template <class Tile>
|
||||
using is_tile = is_tuple<Tile>;
|
||||
|
||||
template <class... Layouts>
|
||||
CUTE_HOST_DEVICE constexpr
|
||||
auto
|
||||
make_tile(Layouts const&... layouts)
|
||||
{
|
||||
return Tile<Layouts...>(layouts...);
|
||||
}
|
||||
|
||||
} // end namespace cute
|
||||
@@ -45,6 +45,9 @@ struct Underscore : Int<0> {};
|
||||
|
||||
CUTE_INLINE_CONSTANT Underscore _;
|
||||
|
||||
// Convenient alias
|
||||
using X = Underscore;
|
||||
|
||||
// Treat Underscore as an integral like integral_constant
|
||||
template <>
|
||||
struct is_integral<Underscore> : true_type {};
|
||||
|
||||
@@ -131,22 +131,21 @@ using CUTE_STL_NAMESPACE::remove_pointer_t;
|
||||
// <utility>
|
||||
using CUTE_STL_NAMESPACE::declval;
|
||||
|
||||
template< class T >
|
||||
template <class T>
|
||||
constexpr T&& forward(remove_reference_t<T>& t) noexcept
|
||||
{
|
||||
return static_cast<T&&>(t);
|
||||
}
|
||||
|
||||
template< class T >
|
||||
template <class T>
|
||||
constexpr T&& forward(remove_reference_t<T>&& t) noexcept
|
||||
{
|
||||
static_assert(! is_lvalue_reference_v<T>,
|
||||
"T cannot be an lvalue reference (e.g., U&).");
|
||||
static_assert(! is_lvalue_reference_v<T>, "T cannot be an lvalue reference (e.g., U&).");
|
||||
return static_cast<T&&>(t);
|
||||
}
|
||||
|
||||
template< class T >
|
||||
constexpr remove_reference_t<T>&& move( T&& t ) noexcept
|
||||
template <class T>
|
||||
constexpr remove_reference_t<T>&& move(T&& t) noexcept
|
||||
{
|
||||
return static_cast<remove_reference_t<T>&&>(t);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user