leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

v4.3 tag release update. (#2789)

Browse Source
This commit is contained in:
Junkai-Wu
2025-11-21 09:49:44 +08:00
committed by GitHub
parent 406e078b29
commit 8cd5bef43a
225 changed files with 23229 additions and 2813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tools/library/src/reference/block_scaled_gemm_fp4a_vs16.cu
View File

@@ -74,6 +74,11 @@ void initialize_block_scaled_gemm_reference_operations_fp4a_vs16(Manifest &manif
16 /*EpilogueSFVecSize*/
>(manifest);
make_block_scaled_gemm_tn<
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 16 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm_tn<
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,

9
tools/library/src/reference/block_scaled_gemm_fp4a_vs32.cu
View File

@@ -73,6 +73,15 @@ void initialize_block_scaled_gemm_reference_operations_fp4a_vs32(Manifest &manif
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
bfloat16_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 32 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 32 /*SFVecSize*/
>(manifest);
// With SF generation reference
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,

18
tools/library/src/reference/gemm_e4m3a_e4m3out.cu
View File

@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e4m3a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e4m3a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB

18
tools/library/src/reference/gemm_e4m3a_e5m2out.cu
View File

@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e4m3a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e4m3a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB

18
tools/library/src/reference/gemm_e5m2a_e4m3out.cu
View File

@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e5m2a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e5m2a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB

9
tools/library/src/reference/gemm_e5m2a_e5m2out.cu
View File

@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e5m2a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB

39
tools/library/src/reference/gemm_f4_f4_f32.cu
View File

@@ -97,7 +97,46 @@ void initialize_gemm_reference_operations_f4_f4_f32(Manifest &manifest) {
float, // ElementAccumulator
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

39
tools/library/src/reference/gemm_f4_f6_f32.cu
View File

@@ -99,6 +99,45 @@ void initialize_gemm_reference_operations_f4_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

79
tools/library/src/reference/gemm_f4_f8_f32.cu
View File

@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
@@ -139,6 +179,45 @@ void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

41
tools/library/src/reference/gemm_f6_f4_f32.cu
View File

@@ -99,6 +99,47 @@ void initialize_gemm_reference_operations_f6_f4_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

40
tools/library/src/reference/gemm_f6_f6_f32.cu
View File

@@ -98,6 +98,46 @@ void initialize_gemm_reference_operations_f6_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

39
tools/library/src/reference/gemm_f6_f8_f32.cu
View File

@@ -99,6 +99,45 @@ void initialize_gemm_reference_operations_f6_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

40
tools/library/src/reference/gemm_f8_f4_f32.cu
View File

@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f8_f4_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

40
tools/library/src/reference/gemm_f8_f6_f32.cu
View File

@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f8_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

61
tools/library/src/reference/gemm_fp32out.cu
View File

@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float // ElementAccumulator
>(manifest);
make_gemm_real_canonical_layouts<
float, // ElementA
float, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
@@ -62,6 +71,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
@@ -69,6 +87,14 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
void,
float,
float,
tfloat32_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
@@ -78,6 +104,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
@@ -86,6 +121,14 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
@@ -94,6 +137,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
@@ -101,6 +153,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

36
tools/library/src/reference/gemm_fp8in_bf16out.cu
View File

@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_bf16out(Manifest &manifest) {
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

36
tools/library/src/reference/gemm_fp8in_fp16out.cu
View File

@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_fp16out(Manifest &manifest) {
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

36
tools/library/src/reference/gemm_fp8in_fp32out.cu
View File

@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_fp32out(Manifest &manifest) {
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

183
tools/library/src/reference/gemm_fp_mixed_input.cu
View File

@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -61,6 +70,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
@@ -68,6 +86,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -75,6 +103,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
@@ -82,6 +119,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -89,6 +135,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -96,6 +151,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -103,6 +167,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -110,6 +183,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -117,6 +200,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -124,6 +216,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -131,6 +232,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
float,
float,
half_t
>(manifest);
// bfloat16_t mixed with 8-bit integer input
make_gemm_real_canonical_layouts<
int8_t,
@@ -139,6 +250,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
@@ -146,6 +266,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
@@ -153,6 +282,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
@@ -160,6 +298,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
@@ -167,6 +314,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
@@ -174,6 +330,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
@@ -181,12 +346,30 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
bfloat16_t,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
void,
float,
float,
bfloat16_t
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

54
tools/library/src/reference/gemm_fp_other.cu
View File

@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
@@ -62,6 +71,24 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
float,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
float,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
double,
double,
@@ -70,6 +97,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
double
>(manifest);
make_gemm_real_canonical_layouts<
double,
double,
void,
double,
double,
double
>(manifest);
make_gemm_complex_canonical_layouts<
complex<float>,
complex<float>,
@@ -78,6 +114,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
complex<float>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<float>,
complex<float>,
void,
complex<float>,
complex<float>,
complex<float>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<double>,
complex<double>,
@@ -85,6 +130,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
complex<double>,
complex<double>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<double>,
complex<double>,
void,
complex<double>,
complex<double>,
complex<double>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

75
tools/library/src/reference/gemm_int4.cu
View File

@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
64,
int4b_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -90,7 +122,28 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
64,
uint4b_t,
uint4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +161,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
uint4b_t,
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -118,6 +182,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

75
tools/library/src/reference/gemm_int8_interleaved_32.cu
View File

@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
32,
int8_t,
int8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
@@ -90,7 +122,28 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
32,
uint8_t,
uint8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +161,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
NumericConverterClamp<uint8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
uint8_t,
NumericConverterClamp<uint8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
@@ -118,6 +182,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

76
tools/library/src/reference/gemm_int8_interleaved_64.cu
View File

@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
64,
int4b_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,18 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -90,7 +123,28 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
64,
uint4b_t,
uint4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +162,17 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
uint4b_t,
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -118,6 +183,17 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

78
tools/library/src/reference/gemm_int_mixed_input.cu
View File

@@ -57,7 +57,26 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
int8_t,
int32_t,
int32_t,
int8_t,
NumericConverterClamp<int8_t, int32_t>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
int32_t,
int32_t,
int8_t,
@@ -77,7 +96,27 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
int8_t,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
float,
int32_t,
int8_t,
@@ -91,6 +130,15 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
@@ -104,7 +152,27 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
int32_t,
int32_t,
int8_t,
NumericConverterClamp<int8_t, int32_t>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -120,6 +188,16 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

11
tools/library/src/reference/gemm_reference_operation.h
View File

@@ -86,7 +86,8 @@ public:
using ElementC = ElementC_;
using LayoutC = LayoutC_;
using ElementD = ElementD_;
using TensorRefC = TensorRef<ElementC, LayoutC>;
using NonVoidElementC = cute::conditional_t<cute::is_void_v<ElementC>, ElementD, ElementC>;
using TensorRefC = TensorRef<NonVoidElementC, LayoutC>;
using TensorRefD = TensorRef<ElementD, LayoutC>;
using ElementCompute = ElementCompute_;
using ElementAccumulator = ElementAccumulator_;
@@ -199,7 +200,7 @@ public:
TensorRefA ref_A{static_cast<ElementA *>(const_cast<void *>(args.A)), LayoutA(int(config.lda))};
TensorRefB ref_B{static_cast<ElementB *>(const_cast<void *>(args.B)), LayoutB(int(config.ldb))};
TensorRefC ref_C{static_cast<ElementC *>(const_cast<void *>(args.C)), LayoutC(int(config.ldc))};
TensorRefC ref_C{static_cast<NonVoidElementC *>(const_cast<void *>(args.C)), LayoutC(int(config.ldc))};
TensorRefD ref_D{static_cast<ElementD *>(args.D), LayoutC(int(config.ldd))};
if (kProvider == Provider::kReferenceHost) {
@@ -209,7 +210,7 @@ public:
LayoutA,
ElementB,
LayoutB,
ElementC,
NonVoidElementC,
LayoutC,
ElementCompute,
ElementAccumulator,
@@ -243,7 +244,7 @@ public:
LayoutA,
ElementB,
LayoutB,
ElementC,
NonVoidElementC,
LayoutC,
ElementCompute,
ElementAccumulator,
@@ -434,7 +435,7 @@ template <
typename ElementCompute_,
typename ElementAccumulator_ = ElementCompute_,
typename ElementD_ = ElementC_,
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
typename ConvertOp_ = NumericConverter<ElementD_, ElementCompute_>,
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
>
void make_gemm_interleaved_layouts(Manifest &manifest) {

74
tools/library/src/reference/gemm_s8_s8_s32.cu
View File

@@ -135,6 +135,80 @@ void initialize_gemm_reference_operations_s8_s8_s32(Manifest &manifest) {
half_t, // ElementD
NumericConverterClamp<half_t, float> // From Scalar to D
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, int32_t> // From Scalar to D
>(manifest);
// 5.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 6.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
float // ElementD
>(manifest);
// 7.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
half_t, // ElementD
NumericConverterClamp<half_t, float> // From Scalar to D
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////

43
tools/library/src/reference/gemm_u8_u8_s32.cu
View File

@@ -98,6 +98,49 @@ void initialize_gemm_reference_operations_u8_u8_s32(Manifest &manifest) {
NumericConverterClamp<uint8_t, float> // From Scalar to D
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t, // ElementD
NumericConverterClamp<int32_t, float> // From Scalar to D
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
uint8_t, // ElementD
NumericConverterClamp<uint8_t, float> // From Scalar to D
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////