leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

v4.5.1 update. (#3237)

Browse Source
This commit is contained in:
Junkai-Wu
2026-05-18 22:35:08 -04:00
committed by GitHub
parent e406c186f5
commit 982cb9e718
42 changed files with 6487 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/cute/arch/mma_sm100_umma.hpp
+8
View File
@@ -1207,6 +1207,10 @@ struct SM100_MMA_S8_2x1SM_SS_SPARSE
}
};
template <class a_type, class b_type, class c_type, int M, int N,
UMMA::Major a_major, UMMA::Major b_major,
UMMA::ScaleIn a_neg = UMMA::ScaleIn::One,
UMMA::ScaleIn b_neg = UMMA::ScaleIn::One>
struct SM100_MMA_F8F6F4_SS
{
using DRegisters = void;
@@ -1452,6 +1456,10 @@ struct SM100_MMA_MXF8F6F4_SS_SPARSE
}
};
template <class a_type, class b_type, class c_type, int M, int N,
UMMA::Major a_major, UMMA::Major b_major,
UMMA::ScaleIn a_neg = UMMA::ScaleIn::One,
UMMA::ScaleIn b_neg = UMMA::ScaleIn::One>
struct SM100_MMA_F8F6F4_2x1SM_SS
{
using DRegisters = void;
include/cute/atom/mma_traits_sm100.hpp
+12 -14
View File
@@ -3327,12 +3327,9 @@ struct MMA_Traits<SM100_MMA_S8_2x1SM_SS_SPARSE<a_type, b_type, c_type,
template <class a_type, class b_type, class c_type,
int M, int N, UMMA::Major a_major, UMMA::Major b_major,
UMMA::ScaleIn a_neg, UMMA::ScaleIn b_neg>
struct MMA_Traits<SM100_MMA_F8F6F4_SS, a_type, b_type, c_type,
cute::C<M>, cute::C<N>,
cute::integral_constant<UMMA::Major, a_major>,
cute::integral_constant<UMMA::Major, b_major>,
cute::integral_constant<UMMA::ScaleIn, a_neg>,
cute::integral_constant<UMMA::ScaleIn, b_neg>>
struct MMA_Traits<SM100_MMA_F8F6F4_SS<a_type, b_type, c_type,
M, N, a_major, b_major,
a_neg, b_neg>>
{
using ValTypeD = c_type;
using ValTypeA = a_type;
@@ -3390,7 +3387,9 @@ struct MMA_Traits<SM100_MMA_F8F6F4_SS, a_type, b_type, c_type,
uint32_t tmem_c = raw_pointer_cast(D.data());
uint64_t idesc = UMMA::make_runtime_instr_desc<>(traits.idesc_);
SM100_MMA_F8F6F4_SS::fma(desc_a, desc_b, tmem_c, uint32_t(traits.accumulate_), idesc);
SM100_MMA_F8F6F4_SS<a_type, b_type, c_type,
M, N, a_major, b_major,
a_neg, b_neg>::fma(desc_a, desc_b, tmem_c, uint32_t(traits.accumulate_), idesc);
}
};
@@ -3745,12 +3744,9 @@ struct MMA_Traits<SM100_MMA_F8F6F4_SS_SPARSE<a_type, b_type, c_type,
template <class a_type, class b_type, class c_type,
int M, int N, UMMA::Major a_major, UMMA::Major b_major,
UMMA::ScaleIn a_neg, UMMA::ScaleIn b_neg>
struct MMA_Traits<SM100_MMA_F8F6F4_2x1SM_SS, a_type, b_type, c_type,
cute::C<M>, cute::C<N>,
cute::integral_constant<UMMA::Major, a_major>,
cute::integral_constant<UMMA::Major, b_major>,
cute::integral_constant<UMMA::ScaleIn, a_neg>,
cute::integral_constant<UMMA::ScaleIn, b_neg>>
struct MMA_Traits<SM100_MMA_F8F6F4_2x1SM_SS<a_type, b_type, c_type,
M, N, a_major, b_major,
a_neg, b_neg>>
{
using ValTypeD = c_type;
using ValTypeA = a_type;
@@ -3808,7 +3804,9 @@ struct MMA_Traits<SM100_MMA_F8F6F4_2x1SM_SS, a_type, b_type, c_type,
uint32_t tmem_c = raw_pointer_cast(D.data());
uint64_t idesc = UMMA::make_runtime_instr_desc<>(traits.idesc_);
SM100_MMA_F8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, uint32_t(traits.accumulate_), idesc);
SM100_MMA_F8F6F4_2x1SM_SS<a_type, b_type, c_type,
M, N, a_major, b_major,
a_neg, b_neg>::fma(desc_a, desc_b, tmem_c, uint32_t(traits.accumulate_), idesc);
}
};
include/cutlass/float8.h
+8
View File
@@ -57,6 +57,14 @@
# endif // (__CUDA_ARCH__ >= 900)
#endif // defined(__CUDA_ARCH__)
#if defined(__CUDA_ARCH__)
# if (__CUDA_ARCH__ >= 1000)
# if (__CUDACC_VER_MAJOR__ > 13) || ((__CUDACC_VER_MAJOR__ >= 13) && (__CUDACC_VER_MINOR__ >= 2))
# define CUDA_PTX_FP8_BF16_CVT_ENABLED 1
# endif // (__CUDACC_VER_MAJOR__ > 13) || ((__CUDACC_VER_MAJOR__ >= 13) && (__CUDACC_VER_MINOR__ >= 2))
# endif // (__CUDA_ARCH__ >= 1000)
#endif // defined(__CUDA_ARCH__)
#if (defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM101A_ENABLED) ||\
defined(CUTLASS_ARCH_MMA_SM103A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM110A_ENABLED) ||\
include/cutlass/gemm/collective/builders/sm100_common.inl
+21 -26
View File
@@ -339,18 +339,16 @@ sm100_make_1sm_trivial_tiled_mma() {
) {
return make_tiled_mma(
cute::MMA_Traits<
cute::SM100_MMA_F8F6F4_SS,
cute::SM100_MMA_F8F6F4_SS<
ElementAMma,
ElementBMma,
ElementAMmaccumulator,
cute::C<M>,
cute::C<N>,
cute::integral_constant<UMMA::Major, UmmaMajorA>,
cute::integral_constant<UMMA::Major, UmmaMajorB>,
cute::integral_constant<UMMA::ScaleIn, ANeg>,
cute::integral_constant<UMMA::ScaleIn, BNeg>
>{}
M,
N,
UmmaMajorA,
UmmaMajorB,
ANeg,
BNeg>{}
);
}
else {
@@ -407,18 +405,16 @@ sm100_make_2sm_trivial_tiled_mma() {
) {
return make_tiled_mma(
cute::MMA_Traits<
cute::SM100_MMA_F8F6F4_2x1SM_SS,
cute::SM100_MMA_F8F6F4_2x1SM_SS<
ElementAMma,
ElementBMma,
ElementAMmaccumulator,
cute::C<M>,
cute::C<N>,
cute::integral_constant<UMMA::Major, UmmaMajorA>,
cute::integral_constant<UMMA::Major, UmmaMajorB>,
cute::integral_constant<UMMA::ScaleIn, ANeg>,
cute::integral_constant<UMMA::ScaleIn, BNeg>
>{}
M,
N,
UmmaMajorA,
UmmaMajorB,
ANeg,
BNeg>{}
);
}
@@ -739,17 +735,16 @@ sm100_make_trivial_mixed_input_tiled_mma() {
}
if constexpr (cute::is_same_v<ElementBMma, cutlass::float_e4m3_t>) {
return make_tiled_mma(
cute::MMA_Traits<
cute::SM100_MMA_F8F6F4_SS,
cute::SM100_MMA_F8F6F4_SS<
ElementAMma,
ElementBMma,
ElementAccumulator,
cute::C<M>,
cute::C<N>,
cute::integral_constant<UMMA::Major, UmmaMajorA>,
cute::integral_constant<UMMA::Major, UmmaMajorB>,
cute::integral_constant<UMMA::ScaleIn, cute::UMMA::ScaleIn::One>,
cute::integral_constant<UMMA::ScaleIn, cute::UMMA::ScaleIn::One>>{});
M,
N,
UmmaMajorA,
UmmaMajorB,
cute::UMMA::ScaleIn::One,
cute::UMMA::ScaleIn::One>{});
}
}
}
include/cutlass/numeric_conversion.h
+38 -19
View File
@@ -1820,14 +1820,21 @@ struct NumericArrayConverter<cutlass::bfloat16_t, cutlass::float_e4m3_t, 2, Roun
#if defined(CUDA_PTX_FP8_CVT_ENABLED)
uint32_t res_half;
uint16_t const& src_packed = reinterpret_cast<uint16_t const&>(source);
asm volatile( \
"{\n" \
"cvt.rn.f16x2.e4m3x2 %0, %1;\n" \
"}\n" : "=r"(res_half): "h"(src_packed));
float2 res_float = __half22float2(reinterpret_cast<__half2 &>(res_half));
NumericArrayConverter<cutlass::bfloat16_t, float, 2, Round> converter;
return converter(reinterpret_cast<Array<float, 2> const&>(res_float));
#if defined(CUDA_PTX_FP8_BF16_CVT_ENABLED)
asm volatile( \
"{\n" \
"cvt.rn.bf16x2.e4m3x2 %0, %1;\n" \
"}\n" : "=r"(res_half): "h"(src_packed));
return reinterpret_cast<result_type const &>(res_half);
#else
asm volatile( \
"{\n" \
"cvt.rn.f16x2.e4m3x2 %0, %1;\n" \
"}\n" : "=r"(res_half): "h"(src_packed));
float2 res_float = __half22float2(reinterpret_cast<__half2 &>(res_half));
NumericArrayConverter<cutlass::bfloat16_t, float, 2, Round> converter;
return converter(reinterpret_cast<Array<float, 2> const&>(res_float));
#endif
#else
result_type result;
NumericConverter<result_element, source_element, Round> converter;
@@ -2961,19 +2968,31 @@ struct NumericArrayConverterPacked4Element<cutlass::bfloat16_t, cutlass::float_e
static result_type convert(source_type const & source) {
#if defined(CUDA_PTX_FP8_CVT_ENABLED)
// Convert f8 to float
NumericArrayConverterPacked4Element<float, source_element, Round> src2float;
Array<float, 4> tmp_floats = src2float(source);
// Convert float to bf16
result_type out;
Array<float, 2>* packed_tmp = reinterpret_cast<Array<float, 2>*>(&tmp_floats);
Array<result_element, 2>* packed_out = reinterpret_cast<Array<result_element, 2>*>(&out);
NumericArrayConverter<result_element, float, 2, Round> float2result;
packed_out[0] = float2result(packed_tmp[0]);
packed_out[1] = float2result(packed_tmp[1]);
#if defined(CUDA_PTX_FP8_BF16_CVT_ENABLED)
uint32_t const& src_packed = reinterpret_cast<uint32_t const&>(source);
Array<uint32_t, 2>& out_packed = reinterpret_cast<Array<uint32_t, 2>&>(out);
asm volatile("{\n"
".reg .b16 b0, b1;\n"
"mov.b32 {b0, b1}, %2;\n"
"cvt.rn.bf16x2.e4m3x2 %0, b0;\n"
"cvt.rn.bf16x2.e4m3x2 %1, b1;\n"
"}\n"
: "=r"(out_packed[0]), "=r"(out_packed[1])
: "r"(src_packed));
#else
// Convert f8 to float
NumericArrayConverterPacked4Element<float, source_element, Round> src2float;
Array<float, 4> tmp_floats = src2float(source);
return out;
// Convert float to bf16
Array<float, 2>* packed_tmp = reinterpret_cast<Array<float, 2>*>(&tmp_floats);
Array<result_element, 2>* packed_out = reinterpret_cast<Array<result_element, 2>*>(&out);
NumericArrayConverter<result_element, float, 2, Round> float2result;
packed_out[0] = float2result(packed_tmp[0]);
packed_out[1] = float2result(packed_tmp[1]);
#endif
return out;
#else
result_type result;
NumericConverter<result_element, source_element, Round> converter;
include/cutlass/version.h
+1 -1
View File
@@ -36,7 +36,7 @@
#define CUTLASS_MAJOR 4
#define CUTLASS_MINOR 5
#define CUTLASS_PATCH 0
#define CUTLASS_PATCH 1
#ifdef CUTLASS_VERSIONS_GENERATED
#include "cutlass/version_extended.h"