leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

CUTLASS 3.6.0 (#1850)

Browse Source 
* v3.6

* update changelog

* update readme

* fix typo

* fixing typos

* hopper gemm with weight prefetch

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2024-10-09 15:33:27 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 0837a2a00a
commit cc3c29a81a
354 changed files with 105937 additions and 8197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/cute/swizzle_layout.hpp
+20 -22
View File
@@ -30,13 +30,10 @@
**************************************************************************************************/
#pragma once
#include <cute/config.hpp>
#include <cute/layout.hpp>
#include <cute/layout_composed.hpp>
#include <cute/swizzle.hpp>
#include <cute/pointer_swizzle.hpp> // get_swizzle
#include <cute/config.hpp> // CUTE_HOST_DEVICE
#include <cute/layout.hpp> // cute::Layout
#include <cute/layout_composed.hpp> // cute::ComposedLayout
#include <cute/swizzle.hpp> // cute::Swizzle, cute::get_swizzle primary template
/* Specialized functionality for a ComposedLayout of the form
* InvolutionFn o Offset o LayoutB
@@ -57,6 +54,9 @@
namespace cute
{
//
// Helper Function
//
template <int B, int M, int S, class Offset, class LayoutB>
struct get_swizzle<ComposedLayout<Swizzle<B,M,S>,Offset,LayoutB>> { using type = Swizzle<B,M,S>; };
@@ -193,7 +193,7 @@ make_swizzle_strides(true_type,
// 0 Z DC
// 1 -Z DC
return cute::make_tuple(conditional_return((offset & (Y << Int<I>{})) == Int<0>{}, Z << Int<I>{}, -(Z << Int<I>{}))...);
return cute::make_tuple(conditional_return((offset & (Y << Int<I>{})) == Int<0>{}, Z * Int<(1 << I)>{}, -Z * Int<(1 << I)>{})...);
}
template <class IntZ, class IntY, class Offset, int... I>
@@ -214,7 +214,7 @@ make_swizzle_strides(false_type,
// 0 Y+Z Y-Z
// 1 DC DC
return cute::make_tuple(conditional_return((offset & (Z << Int<I>{})) == Int<0>{}, (Y+Z) << Int<I>{}, (Y-Z) << Int<I>{})...);
return cute::make_tuple(conditional_return((offset & (Z << Int<I>{})) == Int<0>{}, (Y+Z) * Int<(1 << I)>{}, (Y-Z) * Int<(1 << I)>{})...);
}
} // end namespace detail
@@ -240,16 +240,6 @@ slice_and_offset(Coord const& coord, ComposedLayout<Swizzle<B,M,S>,Offset,Layout
// The portion of the layout that is not yet consumed
auto sliced_layout = slice(coord, layout.layout_b());
// If the sliced_layout hits two bits that are swizzled together, then don't attempt to decay
// Compose with the layout to get the swizzle projection, P o L [The Z and Y contributing portions of L]
// (this also tests that shape/stride of layout compose with swizzle)
auto sliced_layout_only_zy = composition(swizzle_only_zy, sliced_layout);
// Transform the end coordinate to get the active bits of the swizzle, (P o L)(c*)
auto swizzle_active_bits = sliced_layout_only_zy(size(sliced_layout_only_zy)-Int<1>{});
// Determine if any active bits collide under the swizzle
auto hit_ZandY = !(swizzle_active_bits & ~layout.layout_a()(swizzle_active_bits));
// The portion of the layout that we are consuming now
auto diced_layout = dice(coord, layout.layout_b());
auto diced_coord = dice(coord, coord);
@@ -269,8 +259,16 @@ slice_and_offset(Coord const& coord, ComposedLayout<Swizzle<B,M,S>,Offset,Layout
// If Layout's codomain hits on Y XOR Z, then it's dynamic-normal
// If Layout's codomain hits on neither Y NOR Z, then it's static-normal
// Test the sliced layout for hit_X & hit_Y for potential decay
if constexpr (is_constant<false, decltype(hit_ZandY)>::value)
// If the sliced_layout hits two bits that are swizzled together, then don't attempt to decay
// Compose with the layout to get the swizzle projection, P o L [The Z and Y contributing portions of L]
// (this also tests that shape/stride of layout compose with swizzle)
auto sliced_layout_only_zy = composition(swizzle_only_zy, sliced_layout);
// Transform the end coordinate to get the active bits of the swizzle, (P o L)(c*)
[[maybe_unused]] auto swizzle_active_bits = sliced_layout_only_zy(size(sliced_layout_only_zy)-Int<1>{});
// Determine if any active bits collide under the swizzle for potential decay
if constexpr (is_constant<0, decltype(not (swizzle_active_bits & ~swizzle(swizzle_active_bits)))>::value)
{ // Hits on Y AND Z, so it's not reducible
return cute::make_tuple(composition(swizzle, offset_only_zy, sliced_layout), offset_anti_zy);
} else
@@ -459,7 +457,7 @@ CUTE_HOST_DEVICE constexpr
auto
max_alignment(Swizzle<B,M,S> const&)
{
return Int<M>{};
return Int<1 << M>{};
}
template <int B, int M, int S, class Offset, class LayoutB>