leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

v3.9 (#2185)

Browse Source 
* v3.8 update x

* fix blackwell gg

* doc change

* doc change

* doc change

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
This commit is contained in:
Yujia Zhai
2025-03-21 01:52:23 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu Haicheng Wu
parent 8c4d1dc47d
commit 62750a2b75
334 changed files with 91517 additions and 2656 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/cute/core/complement.cpp
+5
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
@@ -68,6 +70,9 @@ test_complement(Layout const& layout, CoTarget const& cotarget)
if constexpr (is_static<decltype(stride(completed))>::value) { // If we can apply complement again
EXPECT_EQ(size(complement(completed)), 1); // There's no more codomain left over
}
if constexpr (is_static<decltype(result)>::value && is_static<decltype(layout)>::value) {
EXPECT_TRUE(bool(complement(complement(result,cosize(layout)),cotarget) == result));
}
}
template <class Layout>
test/unit/cute/core/composition.cpp
+51 -5
View File
@@ -29,6 +29,10 @@
*
**************************************************************************************************/
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cute/layout.hpp>
@@ -38,8 +42,6 @@
#include <cute/tensor.hpp>
#include <iostream>
#include "cutlass_unit_test.h"
using namespace cute;
@@ -273,14 +275,14 @@ TEST(CuTe_core, Composition)
{
auto a = make_layout(Shape<_8,_8>{});
auto b = make_layout(Shape<Shape<_2, _2,_2>, Shape<_2,_2, _2>>{},
auto b = make_layout(Shape <Shape <_2, _2,_2>, Shape <_2,_2, _2>>{},
Stride<Stride<_1,_16,_4>, Stride<_8,_2,_32>>{});
test_composition(a, b);
}
{
auto a = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
auto b = make_layout(Shape<Shape<_2, _2,_2>, Shape<_2,_2, _2>>{},
auto b = make_layout(Shape <Shape <_2, _2,_2>, Shape <_2,_2, _2>>{},
Stride<Stride<_1,_16,_4>, Stride<_8,_2,_32>>{});
test_composition(a, b);
@@ -423,7 +425,7 @@ TEST(CuTe_core, Composition)
test_composition(a, b);
}
// Capping a Layout with 1:0 forces divisibility and extends in stride-0
// Capping a Layout with 1:0 extends in stride-0
{
auto a = make_layout(Shape<_4,_3,_1>{}, Stride<_3,_1,_0>{});
auto b = make_layout(Shape<_24>{});
@@ -431,6 +433,50 @@ TEST(CuTe_core, Composition)
test_composition(a, b);
}
{
auto a = make_layout(Shape<_4,_3,_1>{}, Stride<_3,_1,_0>{});
auto b = make_layout(Shape<_4>{});
test_composition(a, b);
}
// Pre-coalesced LHS
{
auto a = make_layout(Shape<_4,_6,_8>{}, Stride<_1,_4,_7>{});
auto b = make_layout(_6{}, _1{});
test_composition(a, b);
}
// Mid-layout truncation
{
auto a = make_layout(Shape<_4,_6,_8,_10>{}, Stride<_2,_3,_5,_7>{});
auto b = make_layout(_6{}, _12{});
test_composition(a, b);
}
{
auto a = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
auto b = make_layout(_2{}, _3{});
test_composition(a, b);
}
{
auto a = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
auto b = make_layout(_3{}, _3{});
test_composition(a, b);
}
// Should fail to a static divisibility condition
// {
// auto a = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
// auto b = make_layout(_4{}, _3{});
// test_composition(a, b);
// }
{
auto a = make_layout(3, _1{});
auto b = make_layout(_4{}, _1{});
test/unit/cute/core/inverse_left.cpp
+146 -14
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
@@ -50,18 +52,31 @@ test_left_inverse(Layout const& layout)
CUTLASS_TRACE_HOST(layout << " ^ -1\n" << " => \n" << inv_layout);
for (int i = 0; i < size(layout); ++i) {
//printf("%3d: %3d %3d\n", i, int(layout(i)), int(inv_layout(layout(i))));
EXPECT_EQ(inv_layout(layout(i)), i);
//printf("%3d: %3d %3d %3d\n", i, int(layout(i)), int(inv_layout(layout(i))), int(layout(inv_layout(layout(i)))));
EXPECT_EQ(layout(inv_layout(layout(i))), layout(i));
}
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(inv_layout, layout)));
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(layout, composition(inv_layout, layout))));
}
TEST(CuTe_core, Inverse_left)
{
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("LEFT INVERSE" );
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("Simple tests" );
CUTLASS_TRACE_HOST("-------------------------------");
{
auto layout = Layout<Shape <_1>,
Stride<_0>>{};
auto layout = Layout<_1, _0>{};
test_left_inverse(layout);
}
{
auto layout = Layout<_1, _1>{};
test_left_inverse(layout);
}
@@ -74,8 +89,15 @@ TEST(CuTe_core, Inverse_left)
}
{
auto layout = Layout<Shape <_1>,
Stride<_1>>{};
auto layout = Layout<Shape <Shape <_3,_7>>,
Stride<Stride<_0,_0>>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <_4>,
Stride<_0>>{};
test_left_inverse(layout);
}
@@ -94,6 +116,13 @@ TEST(CuTe_core, Inverse_left)
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <_2,_4>,
Stride<_0,_2>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <_8, _4>>{};
@@ -120,6 +149,13 @@ TEST(CuTe_core, Inverse_left)
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <_2,_4,_4,_6>,
Stride<_4,_1,_0,_8>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <_4, _2>,
Stride<_1,_16>>{};
@@ -127,9 +163,105 @@ TEST(CuTe_core, Inverse_left)
test_left_inverse(layout);
}
//
// Swizzle left_inverse
//
{
auto layout = Layout<Shape <_4, _2>,
Stride<_1, _5>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_128>,Stride<_65536,_1>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_160>,Stride<_65536,_1>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_3,_160>,Stride<_65536,_512,_1>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_128, _64>, Stride<Int<131072>, Int<2>>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_32,_4,_4,_4>, Stride<_262144,_4,Int<8388608>,_1>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape<_2,_2,_2>, Stride<_4,_0,_1>>{};
test_left_inverse(layout);
}
{
auto layout = Layout<Shape <Shape <Shape <Shape <Shape < _32, _4>, _1>, Shape < _32, _2>>, _4>, _1, Shape <_2, _2>, _2>,
Stride<Stride<Stride<Stride<Stride<C<262144>, _4>, _0>, Stride<C<0>, C<1>>>, C<8388608>>, _0, Stride<_2, _16>, _32>>{};
test_left_inverse(layout);
}
// CUTLASS_TRACE_HOST("-------------------------------");
// CUTLASS_TRACE_HOST("Dynamic shapes/strides" );
// CUTLASS_TRACE_HOST("-------------------------------");
// {
// auto layout = make_layout(Shape<_4, _2>{}, make_stride(Int<1>{}, 4));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(make_shape(_4{}, 2), make_stride(Int<1>{}, 4));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(make_shape(4, 2), make_stride(Int<1>{}, 4));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(Shape<_2, _4>{}, make_stride(4, Int<1>{}));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(make_shape(2, Int<4>{}), make_stride(4, Int<1>{}));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(make_shape(2, 4), make_stride(4, Int<1>{}));
// test_left_inverse(layout);
// }
// {
// auto layout = make_layout(make_shape(2, 4), make_stride(4, 1));
// test_left_inverse(layout);
// }
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("Swizzle layouts" );
CUTLASS_TRACE_HOST("-------------------------------");
{
auto layout = ComposedLayout<Swizzle<1,0,2>, _0, Layout<Shape <_4, _4>,
@@ -152,10 +284,10 @@ TEST(CuTe_core, Inverse_left)
test_left_inverse(layout);
}
//
// Negative strides (beta support)
// Post-conditions/layout indexing aren't generalized enough to support these yet
// However, the composition post-condition is general enough.
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("BETA: Negative strides" );
CUTLASS_TRACE_HOST("-------------------------------");
{
auto layout = make_layout(Shape<_4>{}, Stride<Int<-1>>{});
test/unit/cute/core/inverse_right.cpp
+71 -21
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
@@ -41,16 +43,6 @@
using namespace cute;
template <class Layout, class InvLayout>
void
test_postconditions(Layout const& layout, InvLayout const& inv_layout)
{
for (int i = 0; i < size(inv_layout); ++i) {
//printf("%3d: %3d %3d\n", i, int(inv_layout(i)), int(layout(inv_layout(i))));
EXPECT_EQ(layout(inv_layout(i)), i);
}
}
template <class Layout>
void
test_right_inverse(Layout const& layout)
@@ -58,9 +50,13 @@ test_right_inverse(Layout const& layout)
auto inv_layout = right_inverse(layout);
CUTLASS_TRACE_HOST(layout << " ^ -1\n" << " => \n" << inv_layout);
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(layout, inv_layout)) << std::endl);
test_postconditions(layout, inv_layout);
for (int i = 0; i < size(inv_layout); ++i) {
//printf("%3d: %3d %3d\n", i, int(inv_layout(i)), int(layout(inv_layout(i))));
EXPECT_EQ(layout(inv_layout(i)), i);
}
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(layout, inv_layout)) << std::endl);
}
TEST(CuTe_core, Inverse_right)
@@ -85,13 +81,6 @@ TEST(CuTe_core, Inverse_right)
test_right_inverse(layout);
}
{
auto layout = Layout<Shape <_4>,
Stride<_0>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape <Shape <_1,_1>>,
Stride<Stride<_0,_0>>>{};
@@ -107,8 +96,8 @@ TEST(CuTe_core, Inverse_right)
}
{
auto layout = Layout<Shape <_1>,
Stride<_1>>{};
auto layout = Layout<Shape <_4>,
Stride<_0>>{};
test_right_inverse(layout);
}
@@ -181,6 +170,49 @@ TEST(CuTe_core, Inverse_right)
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_128>,Stride<_65536,_1>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_160>,Stride<_65536,_1>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_128,_3,_160>,Stride<_65536,_512,_1>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_128, _64>, Stride<Int<131072>, Int<2>>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_32,_4,_4,_4>, Stride<_262144,_4,Int<8388608>,_1>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape<_2,_2,_2>, Stride<_4,_0,_1>>{};
test_right_inverse(layout);
}
{
auto layout = Layout<Shape <Shape <Shape <Shape <Shape < _32, _4>, _1>, Shape < _32, _2>>, _4>, _1, Shape <_2, _2>, _2>,
Stride<Stride<Stride<Stride<Stride<C<262144>, _4>, _0>, Stride<C<0>, C<1>>>, C<8388608>>, _0, Stride<_2, _16>, _32>>{};
test_right_inverse(layout);
}
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("Dynamic shapes/strides" );
CUTLASS_TRACE_HOST("-------------------------------");
@@ -203,6 +235,24 @@ TEST(CuTe_core, Inverse_right)
test_right_inverse(layout);
}
{
auto layout = make_layout(Shape<_2, _4>{}, make_stride(4, Int<1>{}));
test_right_inverse(layout);
}
{
auto layout = make_layout(make_shape(2, Int<4>{}), make_stride(4, Int<1>{}));
test_right_inverse(layout);
}
{
auto layout = make_layout(make_shape(2, 4), make_stride(4, Int<1>{}));
test_right_inverse(layout);
}
CUTLASS_TRACE_HOST("-------------------------------");
CUTLASS_TRACE_HOST("Swizzle layouts" );
CUTLASS_TRACE_HOST("-------------------------------");
test/unit/cute/core/logical_divide.cpp
+2
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
//#define CUTLASS_DEBUG_TRACE_LEVEL 1
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
test/unit/cute/hopper/tma_load_testbed.hpp
+1 -1
View File
@@ -131,7 +131,7 @@ tma_test_device_cute(T const* g_in, T* g_out,
for (int stage = 0; stage < size<1>(tAgA); ++stage)
{
// Set the bytes transferred in this TMA transaction (may involve multiple issues)
constexpr int kTmaTransactionBytes = sizeof(ArrayEngine<T, CUTE_STATIC_V(size(filter_zeros(sA)))>);
constexpr int kTmaTransactionBytes = sizeof(make_tensor_like(tensor<0>(tAsA)));
if (threadIdx.x == 0)
{