@@ -35,6 +35,9 @@
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
#include "cute/layout.hpp"
|
||||
#include "cute/arch/copy_sm90.hpp"
|
||||
|
||||
namespace cutlass {
|
||||
namespace gemm {
|
||||
@@ -420,6 +423,151 @@ enum class SharedMemoryClearOption {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// For each cutlass::layout, provides its corresponding cute stride types, 64b by default
|
||||
|
||||
template <class L>
|
||||
struct TagToStrideA {};
|
||||
|
||||
// Maps to modes [M, K, L]
|
||||
template <>
|
||||
struct TagToStrideA<layout::RowMajor> {
|
||||
using type = cute::Stride<int64_t, cute::Int<1>, int64_t>;
|
||||
using tag = layout::RowMajor;
|
||||
};
|
||||
|
||||
// Maps to modes [M, K, L]
|
||||
template <>
|
||||
struct TagToStrideA<layout::ColumnMajor> {
|
||||
using type = cute::Stride<cute::Int<1>, int64_t, int64_t>;
|
||||
using tag = layout::ColumnMajor;
|
||||
};
|
||||
|
||||
template <class L>
|
||||
struct TagToStrideB {};
|
||||
|
||||
// Maps to modes [N, K, L]
|
||||
template <>
|
||||
struct TagToStrideB<layout::RowMajor> {
|
||||
using type = cute::Stride<cute::Int<1>, int64_t, int64_t>;
|
||||
using tag = layout::RowMajor;
|
||||
};
|
||||
|
||||
// Maps to modes [N, K, L]
|
||||
template <>
|
||||
struct TagToStrideB<layout::ColumnMajor> {
|
||||
using type = cute::Stride<int64_t, cute::Int<1>, int64_t>;
|
||||
using tag = layout::ColumnMajor;
|
||||
};
|
||||
|
||||
|
||||
// Maps to modes [N, N, L]
|
||||
template <class LayoutTag>
|
||||
struct TagToStrideC : TagToStrideA<LayoutTag> { };
|
||||
|
||||
// Convenience aliases
|
||||
template<class LayoutTag>
|
||||
using TagToStrideA_t = typename TagToStrideA<LayoutTag>::type;
|
||||
|
||||
template<class LayoutTag>
|
||||
using TagToStrideB_t = typename TagToStrideB<LayoutTag>::type;
|
||||
|
||||
template<class LayoutTag>
|
||||
using TagToStrideC_t = typename TagToStrideC<LayoutTag>::type;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// For 2.x compatibility APIs, provide stride->layout tag mappers
|
||||
|
||||
namespace detail {
|
||||
|
||||
// Note : This method can be used for deducing the Layout Tag of A, C, D Matrices
|
||||
template<class StrideAC>
|
||||
constexpr
|
||||
auto
|
||||
stride_to_layout_tag_A() {
|
||||
// Account for stride types with and without batch mode and batch modes with static zero stride
|
||||
if constexpr (cute::size<0>(StrideAC{}) == 1) { // M major
|
||||
return layout::ColumnMajor{};
|
||||
}
|
||||
else { // K major
|
||||
return layout::RowMajor{};
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
template<class StrideB>
|
||||
constexpr
|
||||
auto
|
||||
stride_to_layout_tag_B() {
|
||||
// Account for stride types with and without batch mode and batch modes with static zero stride
|
||||
if constexpr (cute::size<0>(StrideB{}) == 1) { // N major
|
||||
return layout::RowMajor{};
|
||||
}
|
||||
else { // K major
|
||||
return layout::ColumnMajor{};
|
||||
}
|
||||
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
}
|
||||
|
||||
// Inspects a TiledCopy and returns its alignment in terms of element count
|
||||
template <class GmemTiledCopy, class Element>
|
||||
constexpr int
|
||||
get_alignment_count_from_gmem_tiled_copy() {
|
||||
// For TMA tiled copies, we know the alignment has to be 128 bits
|
||||
if constexpr (std::is_base_of_v<cute::SM90_TMA_LOAD, GmemTiledCopy> ||
|
||||
std::is_base_of_v<cute::SM90_TMA_LOAD_MULTICAST, GmemTiledCopy>) {
|
||||
return 128 / sizeof_bits<Element>::value;
|
||||
}
|
||||
else
|
||||
{
|
||||
// For non-TMA tiled copies, TiledCopy holds the alignment count directly in its TiledShape_MN
|
||||
return GmemTiledCopy::NumValSrc;
|
||||
}
|
||||
}
|
||||
|
||||
// Utilities to map Stride back on to their corresponding layout tags
|
||||
template <class S>
|
||||
struct StrideToLayoutTagA {
|
||||
using type = decltype(detail::stride_to_layout_tag_A<S>());
|
||||
};
|
||||
|
||||
template <class S>
|
||||
struct StrideToLayoutTagB {
|
||||
using type = decltype(detail::stride_to_layout_tag_B<S>());
|
||||
};
|
||||
|
||||
// Maps to modes [N, N, L]
|
||||
template <class S>
|
||||
struct StrideToLayoutTagC : StrideToLayoutTagA<S> { };
|
||||
|
||||
// Convenience aliases
|
||||
template<class S>
|
||||
using StrideToLayoutTagA_t = typename StrideToLayoutTagA<S>::type;
|
||||
|
||||
template<class S>
|
||||
using StrideToLayoutTagB_t = typename StrideToLayoutTagB<S>::type;
|
||||
|
||||
template<class S>
|
||||
using StrideToLayoutTagC_t = typename StrideToLayoutTagC<S>::type;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// The following two metafunctions are used to detect whether a `kernel::Gemm` or `kernel::GemmUniversal`
|
||||
// is implementing the CUTLASS 3.x API or not, by checking if the problem shape type is aliased within or not.
|
||||
template <class GemmKernel, class = void>
|
||||
struct IsCutlass3GemmKernel : std::false_type { };
|
||||
|
||||
template <typename GemmKernel>
|
||||
struct IsCutlass3GemmKernel<GemmKernel, std::void_t<typename GemmKernel::ProblemShape>>
|
||||
: std::true_type { };
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace detail
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace gemm
|
||||
} // namespace cutlass
|
||||
|
||||
|
||||
Reference in New Issue
Block a user