leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

cutlass 3.9 update (#2255)

Browse Source 
* cutlass 3.9 update

* rebase

* fixes out of shared memory for blockwise Blackwell

* doc format

* fix issue 2253

* disable host ref by default

* fix sm120 smem capacity

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2025-04-24 15:42:40 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 8e345c5c5b
commit 331a1f5b3f
143 changed files with 18089 additions and 5935 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/src/operation_table.cu
+85 -22
View File
@@ -86,6 +86,41 @@ void OperationTable::append(Manifest const &manifest) {
block_scaled_gemm_operations[functional_key][preference_key].push_back(op);
}
if (desc.kind == OperationKind::kBlockwiseGemm) {
BlockwiseGemmDescription const &gemm_desc = static_cast<BlockwiseGemmDescription const &>(desc);
BlockwiseGemmFunctionalKey functional_key(
gemm_desc.provider,
gemm_desc.gemm_kind,
gemm_desc.kind,
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
gemm_desc.A.element,
gemm_desc.A.layout,
gemm_desc.SFA.element,
gemm_desc.B.element,
gemm_desc.B.layout,
gemm_desc.SFB.element,
gemm_desc.C.element,
gemm_desc.C.layout,
gemm_desc.D.element,
gemm_desc.D.layout,
gemm_desc.SFMVecSize,
gemm_desc.SFNVecSize,
gemm_desc.SFKVecSize
);
Operation const *op = operation.get();
int cc = gemm_desc.tile_description.minimum_compute_capability;
int alignment = std::max(std::max(
gemm_desc.A.alignment, gemm_desc.B.alignment), gemm_desc.C.alignment);
GemmPreferenceKey preference_key(cc, alignment);
blockwise_gemm_operations[functional_key][preference_key].push_back(op);
}
// insert all gemm operation into operation table
if (desc.kind == OperationKind::kGemm) {
@@ -157,29 +192,57 @@ void OperationTable::append(Manifest const &manifest) {
}
else {
const BlockScaleDescription &block_scale_desc = grouped_gemm_desc.block_scales.value();
BlockScaledGemmFunctionalKey functional_key(
gemm_desc.provider,
gemm_desc.gemm_kind,
gemm_desc.kind,
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
gemm_desc.A.element,
gemm_desc.A.layout,
block_scale_desc.SFA.element,
gemm_desc.B.element,
gemm_desc.B.layout,
block_scale_desc.SFB.element,
gemm_desc.C.element,
gemm_desc.C.layout,
gemm_desc.D.element,
gemm_desc.D.layout,
block_scale_desc.SFD.element,
block_scale_desc.SFD.layout,
block_scale_desc.SFVecSize,
block_scale_desc.EpilogueSFVecSize
);
if (block_scale_desc.kind == OperationKind::kBlockScaledGemm) {
BlockScaledGemmFunctionalKey functional_key(
gemm_desc.provider,
gemm_desc.gemm_kind,
gemm_desc.kind,
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
gemm_desc.A.element,
gemm_desc.A.layout,
block_scale_desc.SFA.element,
gemm_desc.B.element,
gemm_desc.B.layout,
block_scale_desc.SFB.element,
gemm_desc.C.element,
gemm_desc.C.layout,
gemm_desc.D.element,
gemm_desc.D.layout,
block_scale_desc.SFD.element,
block_scale_desc.SFD.layout,
block_scale_desc.SFKVecSize,
block_scale_desc.EpilogueSFVecSize
);
block_scaled_gemm_operations[functional_key][preference_key].push_back(op);
block_scaled_gemm_operations[functional_key][preference_key].push_back(op);
}
else {
assert(block_scale_desc.kind == OperationKind::kBlockwiseGemm);
BlockwiseGemmFunctionalKey functional_key(
gemm_desc.provider,
gemm_desc.gemm_kind,
gemm_desc.kind,
gemm_desc.tile_description.math_instruction.element_accumulator,
gemm_desc.element_epilogue,
gemm_desc.A.element,
gemm_desc.A.layout,
block_scale_desc.SFA.element,
gemm_desc.B.element,
gemm_desc.B.layout,
block_scale_desc.SFB.element,
gemm_desc.C.element,
gemm_desc.C.layout,
gemm_desc.D.element,
gemm_desc.D.layout,
block_scale_desc.SFMVecSize,
block_scale_desc.SFNVecSize,
block_scale_desc.SFKVecSize
);
blockwise_gemm_operations[functional_key][preference_key].push_back(op);
}
}
}