leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

CUTLASS 3.3.0 (#1167)

Browse Source 
* Release 3.3.0

Adds support for mixed precision GEMMs On Hopper and Ampere
Adds support for < 16B aligned GEMMs on Hopper
Enhancements to EVT
Enhancements to Python interface
Enhancements to Sub-byte type handling in CuTe
Several other bug-fixes and performance improvements.

* minor doc update
This commit is contained in:
Pradeep Ramani
2023-11-02 08:09:05 -07:00
committed by GitHub
parent 922fb5108b
commit c008b4aea8
263 changed files with 16214 additions and 5008 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
include/cute/numeric/arithmetic_tuple.hpp
View File

@@ -126,24 +126,18 @@ operator+(tuple<T...> const& t, ArithmeticTuple<U...> const& u) {
template <auto t, class... U>
CUTE_HOST_DEVICE constexpr
auto
ArithmeticTuple<U...> const&
operator+(C<t>, ArithmeticTuple<U...> const& u) {
if constexpr (t == 0) {
return u;
} else {
static_assert(t == 0, "Artihmetic tuple op+ error!");
}
static_assert(t == 0, "Artihmetic tuple op+ error!");
return u;
}
template <class... T, auto u>
CUTE_HOST_DEVICE constexpr
auto
ArithmeticTuple<T...> const&
operator+(ArithmeticTuple<T...> const& t, C<u>) {
if constexpr (u == 0) {
return t;
} else {
static_assert(u == 0, "Artihmetic tuple op+ error!");
}
static_assert(u == 0, "Artihmetic tuple op+ error!");
return t;
}
//
@@ -153,30 +147,41 @@ operator+(ArithmeticTuple<T...> const& t, C<u>) {
template <class ArithTuple>
struct ArithmeticTupleIterator
{
using value_type = ArithTuple;
using element_type = ArithTuple;
using reference = ArithTuple;
ArithTuple coord_;
CUTE_HOST_DEVICE constexpr
ArithmeticTupleIterator() : coord_() {}
CUTE_HOST_DEVICE constexpr
ArithmeticTupleIterator(ArithTuple const& coord) : coord_(coord) {}
ArithmeticTupleIterator(ArithTuple const& coord = {}) : coord_(coord) {}
CUTE_HOST_DEVICE constexpr
ArithTuple const& operator*() const { return coord_; }
template <class Coord>
CUTE_HOST_DEVICE constexpr
auto operator+(Coord const& c) const {
return ArithmeticTupleIterator<decltype(coord_ + c)>(coord_ + c);
}
auto operator[](Coord const& c) const { return *(*this + c); }
template <class Coord>
CUTE_HOST_DEVICE constexpr
auto operator[](Coord const& c) const { return *(*this + c); }
auto operator+(Coord const& c) const {
return ArithmeticTupleIterator<decltype(coord_ + c)>(coord_ + c);
}
};
template <class ArithTuple>
CUTE_HOST_DEVICE void print(ArithmeticTupleIterator<ArithTuple> const& iter) {
printf("ArithTuple"); print(iter.coord_);
template <class Tuple>
CUTE_HOST_DEVICE constexpr
auto
make_inttuple_iter(Tuple const& t) {
return ArithmeticTupleIterator(as_arithmetic_tuple(t));
}
template <class T0, class T1, class... Ts>
CUTE_HOST_DEVICE constexpr
auto
make_inttuple_iter(T0 const& t0, T1 const& t1, Ts const&... ts) {
return make_tuple_iter(cute::make_tuple(t0, t1, ts...));
}
//
@@ -211,7 +216,7 @@ struct is_integral<ScaledBasis<T,N>> : true_type {};
// Get the scalar T out of a ScaledBasis
template <class SB>
CUTE_HOST_DEVICE constexpr auto
basis_value(SB const& e)
basis_value(SB const& e)
{
if constexpr (is_scaled_basis<SB>::value) {
return basis_value(e.value());
@@ -224,7 +229,7 @@ basis_value(SB const& e)
// Apply the N... pack to another Tuple
template <class SB, class Tuple>
CUTE_HOST_DEVICE constexpr auto
basis_get(SB const& e, Tuple const& t)
basis_get(SB const& e, Tuple const& t)
{
if constexpr (is_scaled_basis<SB>::value) {
return basis_get(e.value(), get<SB::mode()>(t));
@@ -448,36 +453,44 @@ template <auto t, class U, int M>
CUTE_HOST_DEVICE constexpr
auto
operator+(C<t>, ScaledBasis<U,M> const& u) {
if constexpr (t == 0) {
return u;
} else {
static_assert(t == 0, "ScaledBasis op+ error!");
}
static_assert(t == 0, "ScaledBasis op+ error!");
return u;
}
template <class T, int N, auto u>
CUTE_HOST_DEVICE constexpr
auto
operator+(ScaledBasis<T,N> const& t, C<u>) {
if constexpr (u == 0) {
return t;
} else {
static_assert(u == 0, "ScaledBasis op+ error!");
}
static_assert(u == 0, "ScaledBasis op+ error!");
return t;
}
//
// Display utilities
//
template <class ArithTuple>
CUTE_HOST_DEVICE void print(ArithmeticTupleIterator<ArithTuple> const& iter)
{
printf("ArithTuple"); print(iter.coord_);
}
template <class T, int N>
CUTE_HOST_DEVICE void print(ScaledBasis<T,N> const& e) {
CUTE_HOST_DEVICE void print(ScaledBasis<T,N> const& e)
{
print(e.value()); printf("@%d", N);
}
#if !defined(__CUDACC_RTC__)
template <class ArithTuple>
CUTE_HOST std::ostream& operator<<(std::ostream& os, ArithmeticTupleIterator<ArithTuple> const& iter)
{
return os << "ArithTuple" << iter.coord_;
}
template <class T, int N>
CUTE_HOST std::ostream& operator<<(std::ostream& os, ScaledBasis<T,N> const& e) {
CUTE_HOST std::ostream& operator<<(std::ostream& os, ScaledBasis<T,N> const& e)
{
return os << e.value() << "@" << N;
}
#endif

5
include/cute/numeric/int.hpp
View File

@@ -140,6 +140,11 @@ struct sizeof_bits<integer_subbyte<Bits,Signed>> {
static constexpr size_t value = Bits;
};
template <int Bits, bool Signed>
struct sizeof_bits<cutlass::integer_subbyte<Bits,Signed>> {
static constexpr size_t value = Bits;
};
template <class T>
static constexpr int sizeof_bits_v = sizeof_bits<T>::value;

2
include/cute/numeric/integer_subbyte.hpp
View File

@@ -36,6 +36,8 @@
#include <cstdint>
#endif
#include <cutlass/integer_subbyte.h>
#include <cute/config.hpp>
#include <cute/util/type_traits.hpp>

2
include/cute/numeric/integral_constant.hpp
View File

@@ -85,7 +85,7 @@ struct is_integral<integral_constant<T,v>> : true_type {};
// is_static detects if an (abstract) value is defined completely by it's type (no members)
template <class T>
struct is_static : bool_constant<is_empty<T>::value> {};
struct is_static : bool_constant<is_empty<remove_cvref_t<T>>::value> {};
template <class T>
constexpr bool is_static_v = is_static<T>::value;

34
include/cute/numeric/integral_ratio.hpp
View File

@@ -40,12 +40,16 @@ namespace cute
{
/** Compile-time rational arithmetic type.
* Like cute::C for std::integral_constant, cute::R for std::ratio has a short name
* Like cute::C for std::integral_constant, cute::R for std::ratio has a short name
* for error messages and compile times.
* The static data members @a num and @a den represent the reduced numerator and denominator
* of the rational value. Thus, two cute::R types with different @a n or @a d are distinct types
* even if they represent the same rational value. A cute::R exposes the reduced canonical type
* via its type member. That is, cute::R<3,6>::type is cute::R<1,2> and cute::R<6,3>::type is cute::C<2>
* of the rational value. Thus, two cute::R types with different @a n or @a d are distinct types
* even if they represent the same rational value.
* A cute::R exposes the reduced canonical type via its ::type member.
* That is, cute::R<3,6>::type is cute::R<1,2> and cute::R<6,3>::type is cute::C<2>.
* A cute::R<n,d>::value can be used much like any other trait::value. It can be involved in
* arithmetic expressions (according to the operator-overloads for cute::C and cute::R,
* though these may be incomplete) but with a potential rational value rather than an integral value.
*/
template <auto n, auto d>
class R {
@@ -53,7 +57,7 @@ class R {
static constexpr auto an = abs(n);
static constexpr auto ad = abs(d);
static constexpr auto g = gcd(an, ad);
public:
static constexpr auto num = signum(n) * signum(d) * an / g;
static constexpr auto den = ad / g;
@@ -63,28 +67,28 @@ class R {
template <auto a, auto b>
CUTE_HOST_DEVICE constexpr
typename R<a,b>::type
typename R<a,b>::type
ratio(C<a>, C<b>) {
return {};
}
template <auto a, auto b, auto x, auto y>
CUTE_HOST_DEVICE constexpr
typename R<a*x,b*y>::type
typename R<a*x,b*y>::type
operator*(R<a,b>, R<x,y>) {
return {};
}
template <auto a, auto b, auto c>
CUTE_HOST_DEVICE constexpr
typename R<a*c,b>::type
typename R<a*c,b>::type
operator*(R<a,b>, C<c>) {
return {};
}
template <auto c, auto a, auto b>
CUTE_HOST_DEVICE constexpr
typename R<a*c,b>::type
typename R<a*c,b>::type
operator*(C<c>, R<a,b>) {
return {};
}
@@ -109,28 +113,28 @@ operator*(R<a,b>, C const& c) {
template <auto a, auto b, auto x, auto y>
CUTE_HOST_DEVICE constexpr
typename R<a*y+b*x, b*y>::type
typename R<a*y+b*x, b*y>::type
operator+(R<a,b>, R<x,y>) {
return {};
}
template <auto a, auto b, auto c>
CUTE_HOST_DEVICE constexpr
typename R<a+c*b,b>::type
typename R<a+c*b,b>::type
operator+(R<a,b>, C<c>) {
return {};
}
template <auto c, auto a, auto b>
CUTE_HOST_DEVICE constexpr
typename R<a+c*b,b>::type
typename R<a+c*b,b>::type
operator+(C<c>, R<a,b>) {
return {};
}
template <auto a, auto b, auto x, auto y>
CUTE_HOST_DEVICE constexpr
bool_constant<R<a,b>::num == R<x,y>::num && R<a,b>::den == R<x,y>::den>
bool_constant<R<a,b>::num == R<x,y>::num && R<a,b>::den == R<x,y>::den>
operator==(R<a,b>, R<x,y>) {
return {};
}
@@ -144,14 +148,14 @@ operator==(R<a,b>, C<c>) {
template <auto c, auto a, auto b>
CUTE_HOST_DEVICE constexpr
bool_constant<R<a,b>::num == c && R<a,b>::den == 1>
bool_constant<R<a,b>::num == c && R<a,b>::den == 1>
operator==(C<c>, R<a,b>) {
return {};
}
template <auto a, auto b>
CUTE_HOST_DEVICE constexpr
typename R<abs(a),abs(b)>::type
typename R<abs(a),abs(b)>::type
abs(R<a,b>) {
return {};
}

6
include/cute/numeric/math.hpp
View File

@@ -130,6 +130,8 @@ has_single_bit(T x) {
}
// Smallest number of bits needed to represent the given value
// For x == 0, this is 0
// For x != 0, this is 1 + floor(log2(x))
// bit_width( 0b0000 ) = 0
// bit_width( 0b0001 ) = 1
// bit_width( 0b0010 ) = 2
@@ -203,7 +205,7 @@ CUTE_HOST_DEVICE constexpr
T
rotl(T x, int s) {
constexpr int N = numeric_limits<T>::digits;
return s == 0 ? x : s > 0 ? (x << s) | (x >> (N - s)) : rotr(x, -s);
return static_cast<T>(s == 0 ? x : s > 0 ? (x << s) | (x >> (N - s)) : rotr(x, -s));
}
// Computes the result of circular bitwise right-rotation
@@ -212,7 +214,7 @@ CUTE_HOST_DEVICE constexpr
T
rotr(T x, int s) {
constexpr int N = numeric_limits<T>::digits;
return s == 0 ? x : s > 0 ? (x >> s) | (x << (N - s)) : rotl(x, -s);
return static_cast<T>(s == 0 ? x : s > 0 ? (x >> s) | (x << (N - s)) : rotl(x, -s));
}
// Counts the number of consecutive 0 bits, starting from the most significant bit