leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

CUTLASS 3.3.0 (#1167)

Browse Source 
* Release 3.3.0

Adds support for mixed precision GEMMs On Hopper and Ampere
Adds support for < 16B aligned GEMMs on Hopper
Enhancements to EVT
Enhancements to Python interface
Enhancements to Sub-byte type handling in CuTe
Several other bug-fixes and performance improvements.

* minor doc update
This commit is contained in:
Pradeep Ramani
2023-11-02 11:09:05 -04:00
committed by GitHub
parent 922fb5108b
commit c008b4aea8
263 changed files with 16214 additions and 5008 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/core/fast_numeric_conversion.cu
+2 -2
View File
@@ -114,7 +114,7 @@ void run_test_integer_range_all() {
);
destination.sync_host();
// Verify conversion
bool passed = true;
for (int i = 0; i < kN; ++i) {
@@ -124,7 +124,7 @@ void run_test_integer_range_all() {
}
}
EXPECT_TRUE(passed) << " FastNumericArrayConverter failed";
// Print out results for the failed conversion.
if (!passed) {
for (int i = 0; i < kN; ++i) {
test/unit/core/float8.cu
+18
View File
@@ -48,11 +48,20 @@ TEST(float_e4m3_t, host_conversion) {
for (int i = -8; i < 8; ++i) {
float f = static_cast<float>(i);
cutlass::int4b_t s = static_cast<cutlass::int4b_t>(i);
FP8 w = static_cast<FP8>(s);
FP8 x = static_cast<FP8>(i);
FP8 y = static_cast<FP8>(f);
EXPECT_TRUE(static_cast<cutlass::int4b_t>(w) == s);
EXPECT_TRUE(static_cast<int>(x) == i);
EXPECT_TRUE(static_cast<float>(y) == f);
if (i >= 0) {
cutlass::uint4b_t u = static_cast<cutlass::uint4b_t>(i);
FP8 z = static_cast<FP8>(u);
EXPECT_TRUE(static_cast<unsigned>(z) == u);
}
}
// Try out default-ctor (zero initialization of primitive proxy type)
@@ -72,11 +81,20 @@ TEST(float_e5m2_t, host_conversion) {
for (int i = -8; i < 8; ++i) {
float f = static_cast<float>(i);
cutlass::int4b_t s = static_cast<cutlass::int4b_t>(i);
FP8 w = static_cast<FP8>(s);
FP8 x = static_cast<FP8>(i);
FP8 y = static_cast<FP8>(f);
EXPECT_TRUE(static_cast<cutlass::int4b_t>(w) == s);
EXPECT_TRUE(static_cast<int>(x) == i);
EXPECT_TRUE(static_cast<float>(y) == f);
if (i >= 0) {
cutlass::uint4b_t u = static_cast<cutlass::uint4b_t>(i);
FP8 z = static_cast<FP8>(u);
EXPECT_TRUE(static_cast<cutlass::uint4b_t>(z) == u);
}
}
// Try out default-ctor (zero initialization of primitive proxy type)
test/unit/core/numeric_conversion.cu
+123 -8
View File
@@ -60,7 +60,7 @@ __global__ void convert(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Destination, typename Source, int Count>
template <typename Destination, typename Source, int Count, int Range = 4>
void run_test(const char dest_name[], const char source_name[]) {
const int kN = Count;
@@ -69,9 +69,11 @@ void run_test(const char dest_name[], const char source_name[]) {
cutlass::HostTensor<Destination, cutlass::layout::RowMajor> destination({1, kN});
cutlass::HostTensor<Source, cutlass::layout::RowMajor> source({1, kN});
auto source_ref = source.host_ref();
auto destination_ref = destination.host_ref();
for (int i = 0; i < kN; ++i) {
source.host_data()[i] = Source(i % 4);
source_ref.at({0, i}) = Source(i % Range);
}
source.sync_device();
@@ -84,9 +86,67 @@ void run_test(const char dest_name[], const char source_name[]) {
destination.sync_host();
for (int i = 0; i < kN; ++i) {
EXPECT_TRUE(float(destination.host_data()[i]) == float(source.host_data()[i]))
<< "Destination type: " << dest_name
<< ", Source type: " << source_name
EXPECT_TRUE(float(destination_ref.at({0, i})) == float(source_ref.at({0, i})))
<< "Destination type: " << dest_name << " "<< float(destination_ref.at({0, i}))
<< ", Source type: " << source_name << " " << float(source_ref.at({0, i}))
<< ", Count: " << Count;
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Destination, typename Source, typename ScaleFactor, int Count>
__global__ void convert_with_scale_factor(
cutlass::Array<Destination, Count> *destination,
cutlass::Array<Source, Count> const *source,
cutlass::Array<ScaleFactor, Count> const *scale_factor) {
cutlass::NumericArrayConverter<Destination, Source, Count> convert;
*destination = convert(*source, *scale_factor);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Destination, typename Source, typename ScaleFactor, int Count, int Range = 4>
void run_test_with_scalefactor(const char dest_name[], const char source_name[], const char scale_factor_name[]) {
const int kN = Count;
dim3 grid(1, 1);
dim3 block(1, 1);
cutlass::HostTensor<Destination, cutlass::layout::RowMajor> destination({1, kN});
cutlass::HostTensor<Source, cutlass::layout::RowMajor> source({1, kN});
cutlass::HostTensor<ScaleFactor, cutlass::layout::RowMajor> scale_factor({1, kN});
auto source_ref = source.host_ref();
auto destination_ref = destination.host_ref();
auto scale_factor_ref = scale_factor.host_ref();
for (int i = 0; i < kN; ++i) {
source_ref.at({0, i}) = Source(i % Range);
}
for (int i = 0; i < kN; ++i) {
scale_factor_ref.at({0, i}) = ScaleFactor(1 + i % 8);
}
source.sync_device();
scale_factor.sync_device();
convert_with_scale_factor<Destination, Source, ScaleFactor, kN><<< grid, block >>>(
reinterpret_cast<cutlass::Array<Destination, kN> *>(destination.device_data()),
reinterpret_cast<cutlass::Array<Source, kN> const *>(source.device_data()),
reinterpret_cast<cutlass::Array<ScaleFactor, kN> const *>(scale_factor.device_data())
);
destination.sync_host();
for (int i = 0; i < kN; ++i) {
float ref = float(source_ref.at({0, i})) / float(scale_factor_ref.at({0, i}));
EXPECT_TRUE(float(destination_ref.at({0, i})) == ref)
<< "Destination type: " << dest_name << " "<< float(destination_ref.at({0, i}))
<< ", Source type: " << source_name << " " << float(source_ref.at({0, i}))
<< ", Count: " << Count;
}
}
@@ -98,7 +158,16 @@ void run_test(const char dest_name[], const char source_name[]) {
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(NumericConversion, f32_to_f16_rn) {
int const kN = 1;
constexpr int kN = 1;
using Source = float;
const char source_name[] = "float";
using Destination = cutlass::half_t;
const char dest_name[] = "half_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, f32x2_to_f16x2_rn) {
constexpr int kN = 2;
using Source = float;
const char source_name[] = "float";
using Destination = cutlass::half_t;
@@ -107,7 +176,7 @@ TEST(NumericConversion, f32_to_f16_rn) {
}
TEST(NumericConversion, f32x8_to_f16x8_rn) {
int const kN = 8;
constexpr int kN = 8;
using Source = float;
const char source_name[] = "float";
using Destination = cutlass::half_t;
@@ -394,4 +463,50 @@ TEST(NumericConversion, fe5m2_to_bf16_array) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// These are included as regression tests for a special case when N = 4.
TEST(NumericConversion, int4b_t_to_fe5m2_t_array_4) {
int const kN = 4;
using Source = cutlass::int4b_t;
const char source_name[] = "int4b_t";
using Destination = cutlass::float_e5m2_t;
const char dest_name[] = "float_e5m2_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, int_to_fe4m3_t_array_4) {
int const kN = 4;
using Source = int;
const char source_name[] = "int";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, int2b_t_to_fe4m3_t_array_4) {
int const kN = 4;
using Source = cutlass::int2b_t;
const char source_name[] = "int2b_t";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, fe5m2_t_to_double_array_4) {
int const kN = 4;
using Source = cutlass::float_e5m2_t;
const char source_name[] = "float_e5m2_t";
using Destination = double;
const char dest_name[] = "double";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
TEST(NumericConversion, int_to_fe4m3_t_array_32) {
int const kN = 32;
using Source = int;
const char source_name[] = "int";
using Destination = cutlass::float_e4m3_t;
const char dest_name[] = "float_e4m3_t";
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}