leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

v4.4 tag release update. (#3032)

Browse Source
This commit is contained in:
Junkai-Wu
2026-02-14 12:27:58 +08:00
committed by GitHub
parent 01687cfba1
commit d4bbf728ca
140 changed files with 41624 additions and 3691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/cutlass/arch/grid_dependency_control.h
View File

@@ -62,6 +62,8 @@
(defined(__CUDA_ARCH_FEAT_SM100_ALL) || CUDA_ARCH_FAMILY(1000))) || \
(__CUDA_ARCH__ == 1010 &&\
(defined(__CUDA_ARCH_FEAT_SM101_ALL) || CUDA_ARCH_FAMILY(1010))) || \
(__CUDA_ARCH__ == 1100 &&\
(defined(__CUDA_ARCH_FEAT_SM110_ALL) || CUDA_ARCH_FAMILY(1100))) || \
(__CUDA_ARCH__ == 1030 &&\
(defined(__CUDA_ARCH_FEAT_SM103_ALL) || CUDA_ARCH_FAMILY(1030))) || \
(__CUDA_ARCH__ == 1200 &&\

6
include/cutlass/conv/kernel/sm100_implicit_gemm_tma_warpspecialized.hpp
View File

@@ -276,7 +276,11 @@ public:
static constexpr int MaxClusterSize = 16;
implementable &= size(args.hw_info.cluster_shape) <= MaxClusterSize;
implementable &= size(args.hw_info.cluster_shape_fallback) <= MaxClusterSize;
implementable &= cutlass::detail::preferred_cluster_can_implement<AtomThrShapeMNK>(args.hw_info.cluster_shape, args.hw_info.cluster_shape_fallback);
// Early return if cluster shape validation failed to avoid division by zero below
if (not cutlass::detail::preferred_cluster_can_implement<AtomThrShapeMNK>(args.hw_info.cluster_shape, args.hw_info.cluster_shape_fallback)) {
CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Invalid dynamic cluster shape\n");
return false;
}
}
auto cluster_shape = cutlass::detail::select_cluster_shape(ClusterShape{}, args.hw_info.cluster_shape);

5
include/cutlass/gemm/collective/sm100_mma_array_warpspecialized_emulated.hpp
View File

@@ -181,6 +181,11 @@ struct CollectiveMma<
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 64;
static constexpr uint32_t TransformRegisterRequirement = 184;
static constexpr uint32_t AccumRegisterRequirement = 256;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 3;
constexpr static int NumBandsToCompute = DispatchPolicy::NumBandsToCompute;

5
include/cutlass/gemm/collective/sm100_mma_array_warpspecialized_interleaved_complex_emulated.hpp
View File

@@ -189,6 +189,11 @@ public:
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 64;
static constexpr uint32_t TransformRegisterRequirement = 184;
static constexpr uint32_t AccumRegisterRequirement = 256;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 3;
constexpr static int ConjSwapMode = 2;

5
include/cutlass/gemm/collective/sm100_mma_array_warpspecialized_interleaved_complex_tf32.hpp
View File

@@ -173,6 +173,11 @@ public:
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 152;
static constexpr uint32_t TransformRegisterRequirement = 200;
static constexpr uint32_t AccumRegisterRequirement = 152;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 2;
constexpr static int AccumulatorPipelineStageCount = DispatchPolicy::Schedule::AccumulatorPipelineStageCount;

5
include/cutlass/gemm/collective/sm100_mma_warpspecialized_emulated.hpp
View File

@@ -192,6 +192,11 @@ struct CollectiveMma<
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 64;
static constexpr uint32_t TransformRegisterRequirement = 184;
static constexpr uint32_t AccumRegisterRequirement = 256;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 3;
constexpr static int NumBandsToCompute = DispatchPolicy::NumBandsToCompute;

5
include/cutlass/gemm/collective/sm100_mma_warpspecialized_interleaved_complex_emulated.hpp
View File

@@ -185,6 +185,11 @@ public:
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 64;
static constexpr uint32_t TransformRegisterRequirement = 184;
static constexpr uint32_t AccumRegisterRequirement = 256;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 3;
constexpr static int ConjSwapMode = 2;

5
include/cutlass/gemm/collective/sm100_mma_warpspecialized_interleaved_complex_tf32.hpp
View File

@@ -184,6 +184,11 @@ public:
static constexpr uint32_t NumTransformationThreads = 128;
static constexpr uint32_t NumAccumThreads = 128;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = 152;
static constexpr uint32_t TransformRegisterRequirement = 200;
static constexpr uint32_t AccumRegisterRequirement = 152;
// Get the Algorithm parameters
constexpr static int NumComputeMtxs = 2;
constexpr static int AccumulatorPipelineStageCount = DispatchPolicy::Schedule::AccumulatorPipelineStageCount;

37
include/cutlass/gemm/kernel/sm100_gemm_array_tma_warpspecialized_input_transform.hpp
View File

@@ -148,9 +148,10 @@ public:
static constexpr uint32_t NumFixupBarriers = 1;
static constexpr uint32_t CLCResponseSize = sizeof(typename TileScheduler::CLCResponse);
// Transfer registers from regular warps to Accum warps
static constexpr uint32_t GenericRegisterRequirement = 152;
static constexpr uint32_t AccumRegisterRequirement = 200;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = CollectiveMainloop::GenericRegisterRequirement;
static constexpr uint32_t TransformRegisterRequirement = CollectiveMainloop::TransformRegisterRequirement;
static constexpr uint32_t AccumRegisterRequirement = CollectiveMainloop::AccumRegisterRequirement;
// Pipeline and pipeline state types
using Load2TransformPipeline = typename CollectiveMainloop::Load2TransformPipeline;
@@ -412,6 +413,22 @@ public:
return dim3(MaxThreadsPerBlock, 1, 1);
}
// Register alloc/dealloc behavior might change according to the underlying collective used
template <uint32_t NReg>
CUTLASS_DEVICE
static constexpr void
warpgroup_reg_reconfig() {
// Compute default-allocated registers per thread: round_down((512 / NumWG), 8)
constexpr int32_t MaxWarpGroupsPerBlock = ceil_div(MaxThreadsPerBlock, NumThreadsPerWarpGroup);
constexpr int32_t NumRegsPerThread = (512 / MaxWarpGroupsPerBlock) / 8 * 8;
if constexpr (NReg < NumRegsPerThread) {
arch::warpgroup_reg_dealloc<NReg>();
}
else if constexpr (NReg > NumRegsPerThread) {
arch::warpgroup_reg_alloc<NReg>();
}
}
CUTLASS_DEVICE
void
operator() (Params const& params, char* smem_buf) {
@@ -677,7 +694,7 @@ public:
if (is_participant.main_load) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Ensure that the prefetched kernel does not touch
// unflushed global memory prior to this instruction
@@ -791,7 +808,7 @@ public:
else if (is_participant.transformation) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<TransformRegisterRequirement>();
// Signal the epilogue warps to proceed once the prologue is complete
epilogue_throttle_barrier.arrive();
@@ -833,7 +850,7 @@ public:
else if (is_participant.sched) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Signal the epilogue warps to proceed once the prologue is complete
epilogue_throttle_barrier.arrive();
@@ -898,7 +915,7 @@ public:
else if (is_participant.mma) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Allocate all tmem
tmem_allocator.allocate(TmemAllocator::Sm100TmemCapacityColumns, &shared_storage.tmem_base_ptr);
@@ -966,7 +983,7 @@ public:
else if (is_participant.epi_load) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Ensure that the prefetched kernel does not touch
// unflushed global memory prior to this instruction
@@ -1051,7 +1068,7 @@ public:
else if (is_participant.epilogue) {
// Register reconfiguration
arch::warpgroup_reg_alloc<AccumRegisterRequirement>();
warpgroup_reg_reconfig<AccumRegisterRequirement>();
// Throttle the epilogue warps to improve prologue performance
static constexpr int epilogue_throttle_phase_bit = 0;
@@ -1182,7 +1199,7 @@ public:
else {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
}
}
};

38
include/cutlass/gemm/kernel/sm100_gemm_tma_warpspecialized_input_transform.hpp
View File

@@ -143,10 +143,10 @@ public:
static constexpr bool IsSchedDynamicPersistent = TileScheduler::IsDynamicPersistent;
// Transfer registers from regular warps to Accum warps
static constexpr uint32_t GenericRegisterRequirement = 64;
static constexpr uint32_t TransformRegisterRequirement = 184;
static constexpr uint32_t AccumRegisterRequirement = 256;
// Register reconfiguration
static constexpr uint32_t GenericRegisterRequirement = CollectiveMainloop::GenericRegisterRequirement;
static constexpr uint32_t TransformRegisterRequirement = CollectiveMainloop::TransformRegisterRequirement;
static constexpr uint32_t AccumRegisterRequirement = CollectiveMainloop::AccumRegisterRequirement;
// Pipeline and pipeline state types
using Load2TransformPipeline = typename CollectiveMainloop::Load2TransformPipeline;
@@ -389,6 +389,22 @@ public:
return dim3(MaxThreadsPerBlock, 1, 1);
}
// Register alloc/dealloc behavior might change according to the underlying collective used
template <uint32_t NReg>
CUTLASS_DEVICE
static constexpr void
warpgroup_reg_reconfig() {
// Compute default-allocated registers per thread: round_down((512 / NumWG), 8)
constexpr int32_t MaxWarpGroupsPerBlock = ceil_div(MaxThreadsPerBlock, NumThreadsPerWarpGroup);
constexpr int32_t NumRegsPerThread = (512 / MaxWarpGroupsPerBlock) / 8 * 8;
if constexpr (NReg < NumRegsPerThread) {
arch::warpgroup_reg_dealloc<NReg>();
}
else if constexpr (NReg > NumRegsPerThread) {
arch::warpgroup_reg_alloc<NReg>();
}
}
CUTLASS_DEVICE
void
operator() (Params const& params, char* smem_buf) {
@@ -638,7 +654,7 @@ public:
if (is_participant.main_load) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Ensure that the prefetched kernel does not touch
// unflushed global memory prior to this instruction
@@ -716,7 +732,7 @@ public:
else if (is_participant.sched) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Signal the epilogue warps to proceed once the prologue is complete
epilogue_throttle_barrier.arrive();
@@ -770,7 +786,7 @@ public:
else if (is_participant.transformation) {
// Register reconfiguration
arch::warpgroup_reg_alloc<TransformRegisterRequirement>();
warpgroup_reg_reconfig<TransformRegisterRequirement>();
// Signal the epilogue warps to proceed once the prologue is complete
epilogue_throttle_barrier.arrive();
@@ -813,7 +829,7 @@ public:
else if (is_participant.mma) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Tmem allocation sequence
tmem_allocator.allocate(TmemAllocator::Sm100TmemCapacityColumns, &shared_storage.tmem_base_ptr);
@@ -880,7 +896,7 @@ public:
else if (is_participant.epi_load) {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
// Ensure that the prefetched kernel does not touch
// unflushed global memory prior to this instruction
@@ -943,7 +959,7 @@ public:
else if (is_participant.epilogue) {
// Register reconfiguration
arch::warpgroup_reg_alloc<AccumRegisterRequirement>();
warpgroup_reg_reconfig<AccumRegisterRequirement>();
// Throttle the epilogue warps to improve prologue performance
static constexpr int epilogue_throttle_phase_bit = 0;
@@ -1067,7 +1083,7 @@ public:
else {
// Register reconfiguration
arch::warpgroup_reg_dealloc<GenericRegisterRequirement>();
warpgroup_reg_reconfig<GenericRegisterRequirement>();
}
}
};

3
include/cutlass/gemm/kernel/sm100_tile_scheduler_group.hpp
View File

@@ -164,9 +164,6 @@ public:
// Given device SM count, set grid size s.t. we do not launch more thread blocks than we can run concurrently
Arguments args{};
if constexpr (!std::is_const_v<decltype(args.max_swizzle_size)>) {
args.max_swizzle_size = 1 << params.params_sm90_.log_swizzle_size_;
}
args.raster_order = params.params_sm90_.raster_order_ == RasterOrder::AlongN ? RasterOrderOptions::AlongN : RasterOrderOptions::AlongM;
return Params::get_grid_shape(

3
include/cutlass/gemm/kernel/sm90_gemm_array_tma_warpspecialized_cooperative.hpp
View File

@@ -387,9 +387,6 @@ public:
get_grid_shape(Params const& params) {
// Given device SM count, set grid size s.t. we do not launch more thread blocks than we can run concurrently
TileSchedulerArguments args{};
if constexpr (!std::is_const_v<decltype(args.max_swizzle_size)>) {
args.max_swizzle_size = 1 << params.scheduler.log_swizzle_size_;
}
args.raster_order = params.scheduler.raster_order_ == TileScheduler::RasterOrder::AlongN ? TileScheduler::RasterOrderOptions::AlongN : TileScheduler::RasterOrderOptions::AlongM;
dim3 grid_shape;
if constexpr (IsGroupedGemmKernel) {

3
include/cutlass/gemm/kernel/sm90_gemm_array_tma_warpspecialized_pingpong.hpp
View File

@@ -399,9 +399,6 @@ public:
get_grid_shape(Params const& params) {
// Given device SM count, set grid size s.t. we do not launch more thread blocks than we can run concurrently
TileSchedulerArguments args{};
if constexpr (!std::is_const_v<decltype(args.max_swizzle_size)>) {
args.max_swizzle_size = 1 << params.scheduler.log_swizzle_size_;
}
args.raster_order = params.scheduler.raster_order_ == TileScheduler::RasterOrder::AlongN ? TileScheduler::RasterOrderOptions::AlongN : TileScheduler::RasterOrderOptions::AlongM;
dim3 grid_shape;
if constexpr (IsGroupedGemmKernel) {

42
include/cutlass/gemm/kernel/sm90_tile_scheduler_group.hpp
View File

@@ -59,6 +59,7 @@ private:
uint64_t start_linear_idx = 0;
uint64_t total_tiles = 0;
uint64_t problem_blocks_along_raster_order = 0;
int32_t log_swizzle_size = 0;
} current_group_info_;
public:
@@ -244,6 +245,25 @@ public:
return true;
}
// Calculate the log of the swizzle size based on the problem CTAs and the max swizzle size
CUTLASS_DEVICE
static int32_t
get_log_swizzle_size(int problem_ctas_m, int problem_ctas_n, int max_swizzle_size) {
int min_cta_dim = platform::min(problem_ctas_m, problem_ctas_n);
if (max_swizzle_size >= 8 && min_cta_dim >= 6) {
return 3;
}
else if (max_swizzle_size >= 4 && min_cta_dim >= 3) {
return 2;
}
else if (max_swizzle_size >= 2 && min_cta_dim >= 2) {
return 1;
}
else {
return 0;
}
}
PersistentTileSchedulerSm90Group() = default;
// Note: constructing this tile scheduler can touch global memory that was
@@ -276,8 +296,9 @@ public:
ctas_along_m = scheduler_params.divmod_cta_shape_m_.divide(cute::shape<0>(problem_shape) + scheduler_params.divmod_cta_shape_m_.divisor - 1);
ctas_along_n = scheduler_params.divmod_cta_shape_n_.divide(cute::shape<1>(problem_shape) + scheduler_params.divmod_cta_shape_n_.divisor - 1);
}
auto problem_blocks_m = round_up(ctas_along_m, (1 << params_.log_swizzle_size_) * params_.cluster_shape_.m());
auto problem_blocks_n = round_up(ctas_along_n, (1 << params_.log_swizzle_size_) * params_.cluster_shape_.n());
current_group_info_.log_swizzle_size = get_log_swizzle_size(ctas_along_m, ctas_along_n, params_.max_swizzle_size_);
auto problem_blocks_m = round_up(ctas_along_m, (1 << current_group_info_.log_swizzle_size) * params_.cluster_shape_.m());
auto problem_blocks_n = round_up(ctas_along_n, (1 << current_group_info_.log_swizzle_size) * params_.cluster_shape_.n());
current_group_info_.total_tiles = problem_blocks_m * problem_blocks_n;
current_group_info_.problem_blocks_along_raster_order = params_.raster_order_ == RasterOrder::AlongN ? problem_blocks_n : problem_blocks_m;
@@ -302,7 +323,7 @@ public:
FastDivmodU64Pow2 const& divmod_cluster_shape_minor,
FastDivmodU64 const& divmod_cta_shape_m,
FastDivmodU64 const& divmod_cta_shape_n,
int32_t log_swizzle_size,
int32_t max_swizzle_size,
RasterOrder raster_order) {
uint8_t valid_tile = 1;
@@ -310,7 +331,6 @@ public:
// Use a warp to "speculatively" check if the work tile maps to the next 32 groups
int lane_idx = canonical_lane_idx();
int total_problem_groups = problem_shapes.groups();
if (linear_idx >= group_info.total_tiles + group_info.start_linear_idx) {
group_info.group_idx += lane_idx;
for ( ; ; group_info.group_idx += NumThreadsPerWarp) {
@@ -329,8 +349,9 @@ public:
ctas_along_m = divmod_cta_shape_m.divide(cute::shape<0>(cached_problem_shapes[0]) + divmod_cta_shape_m.divisor - 1);
ctas_along_n = divmod_cta_shape_n.divide(cute::shape<1>(cached_problem_shapes[0]) + divmod_cta_shape_n.divisor - 1);
}
auto problem_blocks_m = round_up(ctas_along_m, (1 << log_swizzle_size) * cluster_shape.m());
auto problem_blocks_n = round_up(ctas_along_n, (1 << log_swizzle_size) * cluster_shape.n());
group_info.log_swizzle_size = get_log_swizzle_size(ctas_along_m, ctas_along_n, max_swizzle_size);
auto problem_blocks_m = round_up(ctas_along_m, (1 << group_info.log_swizzle_size) * cluster_shape.m());
auto problem_blocks_n = round_up(ctas_along_n, (1 << group_info.log_swizzle_size) * cluster_shape.n());
group_info.problem_blocks_along_raster_order = raster_order == RasterOrder::AlongN ? problem_blocks_n : problem_blocks_m;
group_info.total_tiles = problem_blocks_m * problem_blocks_n;
}
@@ -356,6 +377,7 @@ public:
group_info.start_linear_idx = __shfl_sync(0xffffffff, group_info.start_linear_idx, first_succeeding_thread);
group_info.total_tiles = __shfl_sync(0xffffffff, group_info.total_tiles, first_succeeding_thread);
group_info.problem_blocks_along_raster_order = __shfl_sync(0xffffffff, group_info.problem_blocks_along_raster_order, first_succeeding_thread);
group_info.log_swizzle_size = __shfl_sync(0xffffffff, group_info.log_swizzle_size, first_succeeding_thread);
if (group_info.group_idx + lane_idx < total_problem_groups) {
cached_problem_shapes[1] = problem_shapes.get_problem_shape(group_info.group_idx + lane_idx);
}
@@ -390,15 +412,15 @@ public:
uint64_t cluster_idx_minor_div_swizzle, extra, offset;
offset = cluster_id & ((1 << log_swizzle_size) - 1);
extra = cluster_id >> log_swizzle_size;
offset = cluster_id & ((1 << group_info.log_swizzle_size) - 1);
extra = cluster_id >> group_info.log_swizzle_size;
uint64_t curr_group_cluster_blk_major = divmod_cluster_shape_major.divide(group_info.problem_blocks_along_raster_order);
cluster_idx_minor_div_swizzle = extra / curr_group_cluster_blk_major;
cluster_idx_major = extra % curr_group_cluster_blk_major;
cluster_idx_minor = cluster_idx_minor_div_swizzle * (1 << log_swizzle_size) + offset;
cluster_idx_minor = cluster_idx_minor_div_swizzle * (1 << group_info.log_swizzle_size) + offset;
auto minor_work_idx = static_cast<int32_t>(cluster_idx_minor * divmod_cluster_shape_minor.divisor +
cluster_minor_offset);
@@ -430,7 +452,7 @@ public:
scheduler_params.divmod_cluster_shape_minor_,
scheduler_params.divmod_cta_shape_m_,
scheduler_params.divmod_cta_shape_n_,
scheduler_params.log_swizzle_size_,
scheduler_params.max_swizzle_size_,
scheduler_params.raster_order_);
}

5
include/cutlass/gemm/kernel/sm90_tile_scheduler_stream_k.hpp
View File

@@ -246,8 +246,9 @@ public:
static bool
can_implement(Arguments const& args) {
// Split count > 1 is only valid for heuristic and split-K decomposition modes
return (args.splits == 1 ||
// Split count must be positive, and > 1 is only valid for heuristic and split-K decomposition modes
return args.splits >= 1 &&
(args.splits == 1 ||
args.decomposition_mode == DecompositionMode::Heuristic ||
args.decomposition_mode == DecompositionMode::SplitK);
}

46
include/cutlass/gemm/kernel/tile_scheduler_params.h
View File

@@ -1635,7 +1635,7 @@ struct PersistentTileSchedulerSm90GroupParams {
uint64_t blocks_across_problem_ = 0;
bool pre_processed_problem_shapes = true;
int32_t log_swizzle_size_ = 0;
int32_t max_swizzle_size_ = 0;
RasterOrder raster_order_ = RasterOrder::AlongN;
GroupProblemShape problem_shapes_;
@@ -1658,10 +1658,8 @@ struct PersistentTileSchedulerSm90GroupParams {
CUTLASS_UNUSED(hw_info);
// Round up to nearest multiple of swizzle_size along each mode
auto log_swizzle_size = get_log_swizzle_size(problem_blocks.x, problem_blocks.y, max_swizzle_size);
auto problem_blocks_m = round_up(problem_blocks.x, (1 << log_swizzle_size) * cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, (1 << log_swizzle_size) * cluster_shape.n());
auto problem_blocks_m = round_up(problem_blocks.x, cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, cluster_shape.n());
RasterOrder raster_order = get_rasterization_order(
problem_blocks_m,
@@ -1678,7 +1676,7 @@ struct PersistentTileSchedulerSm90GroupParams {
blocks_across_problem_ = problem_blocks.x * problem_blocks.y * problem_blocks.z;
pre_processed_problem_shapes = problem_shapes.is_host_problem_shape_available();
log_swizzle_size_ = log_swizzle_size;
max_swizzle_size_ = max_swizzle_size;
raster_order_ = raster_order;
if (raster_order == RasterOrder::AlongN) {
@@ -1727,10 +1725,8 @@ struct PersistentTileSchedulerSm90GroupParams {
int const sm_count = hw_info.sm_count;
int const max_active_clusters = hw_info.max_active_clusters;
// Round up to nearest multiple of swizzle_size along each mode
auto log_swizzle_size = get_log_swizzle_size(problem_blocks.x, problem_blocks.y, max_swizzle_size);
auto problem_blocks_m = round_up(problem_blocks.x, (1 << log_swizzle_size) * cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, (1 << log_swizzle_size) * cluster_shape.n());
auto problem_blocks_m = round_up(problem_blocks.x, cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, cluster_shape.n());
int problem_blocks_total = problem_blocks_m * problem_blocks_n * problem_blocks.z;
@@ -1803,24 +1799,6 @@ struct PersistentTileSchedulerSm90GroupParams {
return launch_grid;
}
CUTLASS_HOST_DEVICE
static int32_t
get_log_swizzle_size(int problem_ctas_m, int problem_ctas_n, int max_swizzle_size) {
int min_cta_dim = platform::min(problem_ctas_m, problem_ctas_n);
if (max_swizzle_size >= 8 && min_cta_dim >= 6) {
return 3;
}
else if (max_swizzle_size >= 4 && min_cta_dim >= 3) {
return 2;
}
else if (max_swizzle_size >= 2 && min_cta_dim >= 2) {
return 1;
}
else {
return 0;
}
}
CUTLASS_HOST_DEVICE
static RasterOrder
get_rasterization_order(
@@ -2496,10 +2474,8 @@ struct PersistentTileSchedulerSm100GroupParams {
int const sm_count = hw_info.sm_count;
int const max_active_clusters = hw_info.max_active_clusters;
// Round up to nearest multiple of swizzle_size along each mode
auto log_swizzle_size = get_log_swizzle_size(problem_blocks.x, problem_blocks.y, max_swizzle_size);
auto problem_blocks_m = round_up(problem_blocks.x, (1 << log_swizzle_size) * cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, (1 << log_swizzle_size) * cluster_shape.n());
auto problem_blocks_m = round_up(problem_blocks.x, cluster_shape.m());
auto problem_blocks_n = round_up(problem_blocks.y, cluster_shape.n());
int problem_blocks_total = problem_blocks_m * problem_blocks_n * problem_blocks.z;
@@ -2581,12 +2557,6 @@ struct PersistentTileSchedulerSm100GroupParams {
return launch_grid;
}
CUTLASS_HOST_DEVICE
static int32_t
get_log_swizzle_size(int problem_ctas_m, int problem_ctas_n, int max_swizzle_size) {
return UnderlyingSm90Params::get_log_swizzle_size(problem_ctas_m, problem_ctas_n, max_swizzle_size);
}
CUTLASS_HOST_DEVICE
static RasterOrder
get_rasterization_order(

12
include/cutlass/platform/platform.h
View File

@@ -582,6 +582,7 @@ template <typename value_t>
struct alignment_of : std::alignment_of<value_t> {};
#endif
#if CUDA_VERSION >= 11080
/* 16B specializations where 32-bit Win32 host compiler disagrees with device compiler */
template <>
@@ -609,12 +610,7 @@ struct alignment_of<double2> {
enum { value = 16 };
};
#if !defined(CUDA_VECTOR_TYPE_ALIGNMENT_16_32_ENABLED)
#define CUDA_VECTOR_TYPE_ALIGNMENT_16_32_ENABLED (__CUDACC_VER_MAJOR__ >= 13)
#endif
#if (CUDA_VECTOR_TYPE_ALIGNMENT_16_32_ENABLED)
#if CUDA_VERSION >= 13000
template <>
struct alignment_of<long4_16a> {
enum { value = 16 };
@@ -655,7 +651,9 @@ template <>
struct alignment_of<double4_32a> {
enum { value = 32 };
};
#else
template <>
struct alignment_of<long4> {
enum { value = 16 };
@@ -677,7 +675,7 @@ struct alignment_of<double4> {
enum { value = 16 };
};
#endif
#endif // CUDA_VERSION >= 13000
#endif // CUDA_VERSION >= 11080
// Specializations for volatile/const qualified types