CUTLASS 3.0.0 (#786)

* CUTLASS 3.0.0
This commit is contained in:
Vijay Thakkar
2023-01-23 20:55:28 -05:00
committed by GitHub
parent 66d9cddc83
commit 277bd6e537
377 changed files with 76396 additions and 1186 deletions
+148
View File
@@ -35,6 +35,9 @@
#include "cutlass/cutlass.h"
#include "cutlass/coord.h"
#include "cutlass/layout/matrix.h"
#include "cute/layout.hpp"
#include "cute/arch/copy_sm90.hpp"
namespace cutlass {
namespace gemm {
@@ -420,6 +423,151 @@ enum class SharedMemoryClearOption {
////////////////////////////////////////////////////////////////////////////////////////////////////
// For each cutlass::layout, provides its corresponding cute stride types, 64b by default
template <class L>
struct TagToStrideA {};
// Maps to modes [M, K, L]
template <>
struct TagToStrideA<layout::RowMajor> {
using type = cute::Stride<int64_t, cute::Int<1>, int64_t>;
using tag = layout::RowMajor;
};
// Maps to modes [M, K, L]
template <>
struct TagToStrideA<layout::ColumnMajor> {
using type = cute::Stride<cute::Int<1>, int64_t, int64_t>;
using tag = layout::ColumnMajor;
};
template <class L>
struct TagToStrideB {};
// Maps to modes [N, K, L]
template <>
struct TagToStrideB<layout::RowMajor> {
using type = cute::Stride<cute::Int<1>, int64_t, int64_t>;
using tag = layout::RowMajor;
};
// Maps to modes [N, K, L]
template <>
struct TagToStrideB<layout::ColumnMajor> {
using type = cute::Stride<int64_t, cute::Int<1>, int64_t>;
using tag = layout::ColumnMajor;
};
// Maps to modes [N, N, L]
template <class LayoutTag>
struct TagToStrideC : TagToStrideA<LayoutTag> { };
// Convenience aliases
template<class LayoutTag>
using TagToStrideA_t = typename TagToStrideA<LayoutTag>::type;
template<class LayoutTag>
using TagToStrideB_t = typename TagToStrideB<LayoutTag>::type;
template<class LayoutTag>
using TagToStrideC_t = typename TagToStrideC<LayoutTag>::type;
////////////////////////////////////////////////////////////////////////////////////////////////////
// For 2.x compatibility APIs, provide stride->layout tag mappers
namespace detail {
// Note : This method can be used for deducing the Layout Tag of A, C, D Matrices
template<class StrideAC>
constexpr
auto
stride_to_layout_tag_A() {
// Account for stride types with and without batch mode and batch modes with static zero stride
if constexpr (cute::size<0>(StrideAC{}) == 1) { // M major
return layout::ColumnMajor{};
}
else { // K major
return layout::RowMajor{};
}
CUTE_GCC_UNREACHABLE;
}
template<class StrideB>
constexpr
auto
stride_to_layout_tag_B() {
// Account for stride types with and without batch mode and batch modes with static zero stride
if constexpr (cute::size<0>(StrideB{}) == 1) { // N major
return layout::RowMajor{};
}
else { // K major
return layout::ColumnMajor{};
}
CUTE_GCC_UNREACHABLE;
}
// Inspects a TiledCopy and returns its alignment in terms of element count
template <class GmemTiledCopy, class Element>
constexpr int
get_alignment_count_from_gmem_tiled_copy() {
// For TMA tiled copies, we know the alignment has to be 128 bits
if constexpr (std::is_base_of_v<cute::SM90_TMA_LOAD, GmemTiledCopy> ||
std::is_base_of_v<cute::SM90_TMA_LOAD_MULTICAST, GmemTiledCopy>) {
return 128 / sizeof_bits<Element>::value;
}
else
{
// For non-TMA tiled copies, TiledCopy holds the alignment count directly in its TiledShape_MN
return GmemTiledCopy::NumValSrc;
}
}
// Utilities to map Stride back on to their corresponding layout tags
template <class S>
struct StrideToLayoutTagA {
using type = decltype(detail::stride_to_layout_tag_A<S>());
};
template <class S>
struct StrideToLayoutTagB {
using type = decltype(detail::stride_to_layout_tag_B<S>());
};
// Maps to modes [N, N, L]
template <class S>
struct StrideToLayoutTagC : StrideToLayoutTagA<S> { };
// Convenience aliases
template<class S>
using StrideToLayoutTagA_t = typename StrideToLayoutTagA<S>::type;
template<class S>
using StrideToLayoutTagB_t = typename StrideToLayoutTagB<S>::type;
template<class S>
using StrideToLayoutTagC_t = typename StrideToLayoutTagC<S>::type;
///////////////////////////////////////////////////////////////////////////////
// The following two metafunctions are used to detect whether a `kernel::Gemm` or `kernel::GemmUniversal`
// is implementing the CUTLASS 3.x API or not, by checking if the problem shape type is aliased within or not.
template <class GemmKernel, class = void>
struct IsCutlass3GemmKernel : std::false_type { };
template <typename GemmKernel>
struct IsCutlass3GemmKernel<GemmKernel, std::void_t<typename GemmKernel::ProblemShape>>
: std::true_type { };
///////////////////////////////////////////////////////////////////////////////
} // namespace detail
///////////////////////////////////////////////////////////////////////////////
} // namespace gemm
} // namespace cutlass