leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/cute/numeric/arithmetic_tuple.hpp
+6 -6
View File
@@ -197,7 +197,7 @@ struct ArithmeticTupleIterator
ArithmeticTupleIterator(ArithTuple const& coord = {}) : coord_(coord) {}
CUTE_HOST_DEVICE constexpr
ArithTuple const& operator*() const { return coord_; }
ArithTuple operator*() const { return coord_; }
template <class Coord>
CUTE_HOST_DEVICE constexpr
@@ -206,7 +206,7 @@ struct ArithmeticTupleIterator
template <class Coord>
CUTE_HOST_DEVICE constexpr
auto operator+(Coord const& c) const {
return ArithmeticTupleIterator<decltype(coord_ + c)>(coord_ + c);
return ArithmeticTupleIterator<remove_cvref_t<decltype(coord_ + c)>>(coord_ + c);
}
};
@@ -268,13 +268,13 @@ basis_value(SB const& e)
// Apply the N... pack to another Tuple
template <class SB, class Tuple>
CUTE_HOST_DEVICE constexpr auto
basis_get(SB const& e, Tuple const& t)
CUTE_HOST_DEVICE decltype(auto)
basis_get(SB const& e, Tuple&& t)
{
if constexpr (is_scaled_basis<SB>::value) {
return basis_get(e.value(), get<SB::mode()>(t));
return basis_get(e.value(), get<SB::mode()>(static_cast<Tuple&&>(t)));
} else {
return t;
return static_cast<Tuple&&>(t);
}
CUTE_GCC_UNREACHABLE;
}
include/cute/numeric/complex.hpp
+3 -3
View File
@@ -30,9 +30,9 @@
**************************************************************************************************/
#pragma once
#include <cutlass/complex.h>
#include <cute/util/type_traits.hpp>
#include <cute/numeric/numeric_types.hpp>
#include <cute/config.hpp> // CUTE_HOST_DEVICE
#include <cutlass/complex.h> // cutlass::complexm, cutlass::real, cutlass::imag, cutlass::is_complex
namespace cute
{
include/cute/numeric/int.hpp
+8 -6
View File
@@ -36,7 +36,9 @@
#include <cstdint>
#endif
#include <cutlass/numeric_types.h>
#include <cute/config.hpp> // CUTE_STL_NAMESPACE
#include <cutlass/numeric_types.h> // cutlass::int2b_t, cutlass::int4b_t
namespace cute
{
@@ -53,8 +55,8 @@ using CUTE_STL_NAMESPACE::int32_t;
using CUTE_STL_NAMESPACE::int64_t;
template <int N> struct int_bit;
template <> struct int_bit< 2> { using type = cutlass::int2b_t; };
template <> struct int_bit< 4> { using type = cutlass::int4b_t; };
template <> struct int_bit< 2> { using type = int2_t; };
template <> struct int_bit< 4> { using type = int4_t; };
template <> struct int_bit< 8> { using type = int8_t; };
template <> struct int_bit< 16> { using type = int16_t; };
template <> struct int_bit< 32> { using type = int32_t; };
@@ -83,9 +85,9 @@ using CUTE_STL_NAMESPACE::uint64_t;
using cutlass::uint128_t;
template <int N> struct uint_bit;
template <> struct uint_bit< 1> { using type = cutlass::uint1b_t; };
template <> struct uint_bit< 2> { using type = cutlass::uint2b_t; };
template <> struct uint_bit< 4> { using type = cutlass::uint4b_t; };
template <> struct uint_bit< 1> { using type = uint1_t; };
template <> struct uint_bit< 2> { using type = uint2_t; };
template <> struct uint_bit< 4> { using type = uint4_t; };
template <> struct uint_bit< 8> { using type = uint8_t; };
template <> struct uint_bit< 16> { using type = uint16_t; };
template <> struct uint_bit< 32> { using type = uint32_t; };
include/cute/numeric/integral_constant.hpp
+19 -6
View File
@@ -30,10 +30,9 @@
**************************************************************************************************/
#pragma once
#include "cute/util/print.hpp"
#include "cute/util/type_traits.hpp"
#include "cute/numeric/math.hpp"
#include "cutlass/fast_math.h"
#include <cute/numeric/math.hpp> // cute::max, etc
#include <cute/util/print.hpp> // cute::print
#include <cute/util/type_traits.hpp> // __CUTE_REQUIRES, cute::is_std_integral
namespace cute
{
@@ -65,7 +64,7 @@ struct integral_constant : C<v> {
static constexpr T value = v;
using value_type = T;
// Disambiguate C<v>::operator value_type()
//CUTE_HOST_DEVICE constexpr operator value_type() const noexcept { return value; }
//CUTE_HOST_DEVICE constexpr operator value_type() const noexcept { return value; }
CUTE_HOST_DEVICE constexpr value_type operator()() const noexcept { return value; }
};
@@ -406,6 +405,20 @@ conditional_return(false_type, TrueType&&, FalseType&& f) {
return static_cast<FalseType&&>(f);
}
template <auto v>
CUTE_HOST_DEVICE constexpr
auto
conditional_return(bool b, C<v> const&, C<v> const&) {
return C<v>{};
}
template <auto v, auto u>
CUTE_HOST_DEVICE constexpr
auto
conditional_return(bool b, C<v> const&, C<u> const&) {
return b ? v : u;
}
// TrueType and FalseType must have a common type
template <class TrueType, class FalseType>
CUTE_HOST_DEVICE constexpr
@@ -435,7 +448,7 @@ static_value()
return Int<Trait::value>{};
} else {
return Trait::value;
}
}
CUTE_GCC_UNREACHABLE;
}
include/cute/numeric/integral_ratio.hpp
+4 -5
View File
@@ -30,11 +30,10 @@
**************************************************************************************************/
#pragma once
#include <cute/config.hpp>
#include <cute/util/type_traits.hpp>
#include <cute/numeric/math.hpp>
#include <cute/numeric/integral_constant.hpp>
#include <cute/config.hpp> // CUTE_HOST_DEVICE
#include <cute/numeric/integral_constant.hpp> // cute::false_type, cute::true_type
#include <cute/numeric/math.hpp> // cute::signum
#include <cute/util/type_traits.hpp> // __CUTE_REQUIRES
namespace cute
{
include/cute/numeric/math.hpp
+9 -9
View File
@@ -30,9 +30,9 @@
**************************************************************************************************/
#pragma once
#include <cute/config.hpp>
#include <cute/config.hpp> // CUTE_HOST_DEVICE
#include <cute/util/type_traits.hpp> // __CUTE_REQUIRES
#include <cute/util/type_traits.hpp>
#include <cutlass/fast_math.h>
namespace cute
@@ -143,7 +143,7 @@ has_single_bit(T x) {
// bit_width( 0b0111 ) = 3
template <class T>
CUTE_HOST_DEVICE constexpr
T
int
bit_width(T x) {
static_assert(is_unsigned<T>::value, "Only to be used for unsigned types.");
constexpr int N = (numeric_limits<T>::digits == 64 ? 6 :
@@ -224,7 +224,7 @@ rotr(T x, int s) {
// countl_zero( 0b00011100 ) = 3
template <class T>
CUTE_HOST_DEVICE constexpr
T
int
countl_zero(T x) {
return numeric_limits<T>::digits - bit_width(x);
}
@@ -235,7 +235,7 @@ countl_zero(T x) {
// countl_one( 0b11100011 ) = 3
template <class T>
CUTE_HOST_DEVICE constexpr
T
int
countl_one(T x) {
return countl_zero(~x);
}
@@ -246,7 +246,7 @@ countl_one(T x) {
// countr_zero( 0b00011100 ) = 2
template <class T>
CUTE_HOST_DEVICE constexpr
T
int
countr_zero(T x) {
return x == 0 ? numeric_limits<T>::digits : bit_width(T(x & T(-x))) - 1; // bit_width of the LSB
}
@@ -257,7 +257,7 @@ countr_zero(T x) {
// countr_one( 0b11100011 ) = 2
template <class T>
CUTE_HOST_DEVICE constexpr
T
int
countr_one(T x) {
return countr_zero(~x);
}
@@ -285,7 +285,7 @@ popcount(T x) {
// Computes the result of bitwise left-shift
template <class T>
CUTE_HOST_DEVICE constexpr
T
auto
shiftl(T x, int s) {
return s >= 0 ? (x << s) : (x >> -s);
}
@@ -293,7 +293,7 @@ shiftl(T x, int s) {
// Computes the result of bitwise right-shift
template <class T>
CUTE_HOST_DEVICE constexpr
T
auto
shiftr(T x, int s) {
return s >= 0 ? (x >> s) : (x << -s);
}
include/cute/numeric/numeric_types.hpp
+66 -6
View File
@@ -30,12 +30,11 @@
**************************************************************************************************/
#pragma once
#include <vector_types.h>
#include <cutlass/numeric_types.h>
#include <cutlass/numeric_size.h>
#include <cute/config.hpp> // CUTE_HOST_DEVICE
#include <cute/numeric/int.hpp> // cute::int2_t, cute::int4_t, etc
#include <cute/numeric/int.hpp>
#include <cute/numeric/real.hpp>
#include <cutlass/numeric_size.h> // cutlass::sizeof_bits
#include <cutlass/numeric_types.h> // cutlass::float_e4m3_t, cutlass::float_e5m2_t, etc
namespace cute {
@@ -72,4 +71,65 @@ using cutlass::int4b_t;
using cutlass::uint4b_t;
using cutlass::bin1_t;
} // end namespace cute
//
// Print utility
//
CUTE_HOST_DEVICE
void
print(half_t a) {
printf("%f", static_cast<float>(a));
}
CUTE_HOST_DEVICE
void
print(bfloat16_t a) {
printf("%f", static_cast<float>(a));
}
CUTE_HOST_DEVICE
void
print(tfloat32_t a) {
printf("%f", static_cast<float>(a));
}
CUTE_HOST_DEVICE
void
print(float_e4m3_t a) {
printf("%f", static_cast<float>(a));
}
CUTE_HOST_DEVICE
void
print(float_e5m2_t a) {
printf("%f", static_cast<float>(a));
}
CUTE_HOST_DEVICE void
pretty_print(bfloat16_t v) {
printf("%*.2f", 8, float(v));
}
CUTE_HOST_DEVICE void
pretty_print(half_t v) {
printf("%*.2f", 8, float(v));
}
CUTE_HOST_DEVICE void
pretty_print(tfloat32_t v) {
printf("%*.2e", 10, static_cast<float>(v));
}
CUTE_HOST_DEVICE void
pretty_print(float_e4m3_t t) {
printf("%*.2f", 8, static_cast<float>(t));
}
CUTE_HOST_DEVICE void
pretty_print(float_e5m2_t t) {
printf("%*.2f", 8, static_cast<float>(t));
}
} // namespace cute
include/cute/numeric/real.hpp
+18
View File
@@ -35,6 +35,24 @@
namespace cute
{
/// Generic add
template <class C, class A, class B>
CUTE_HOST_DEVICE constexpr
void
add(C& c, A const& a, B const& b)
{
c = a + b;
}
/// Generic multiply
template <class C, class A, class B>
CUTE_HOST_DEVICE constexpr
void
mul(C& c, A const& a, B const& b)
{
c = a * b;
}
/// Generic fused multiply-add
template <class D, class A, class B, class C>
CUTE_HOST_DEVICE constexpr