leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

CUTLASS 3.4.0 (#1286)

Browse Source 
* CUTLASS 3.4.0

* Update CHANGELOG.md

---------

Co-authored-by: Pradeep Ramani <prramani@nvidia.com>
This commit is contained in:
Pradeep Ramani
2023-12-29 15:21:31 -05:00
committed by GitHub
co-authored by Pradeep Ramani
parent b7508e3379
commit 8236f30675
211 changed files with 11409 additions and 2763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/python/cutlass/evt/evt_compute_sm80_90.py
+2 -2
View File
@@ -46,8 +46,8 @@ from utils.evt_testbed import EVTTestBed, EVTTestCaseBase
cutlass.set_log_level(logging.WARNING)
@unittest.skipIf(device_cc() not in [80, 90], "This unittest is for Sm80 and Sm90 only")
class TestEVTComputeSM90(EVTTestCaseBase):
@unittest.skipIf(device_cc() not in [80, 86, 89, 90], "This unittest is only supported on CC [80, 86, 89, 90]")
class TestEVTCompute(EVTTestCaseBase):
def test_arith(self):
"""
test/python/cutlass/evt/evt_layout_sm80_90.py
+4 -4
View File
@@ -46,8 +46,8 @@ from utils.evt_testbed import EVTTestBed, EVTTestCaseBase
cutlass.set_log_level(logging.WARNING)
@unittest.skipIf(device_cc() not in [80, 90], "This unittest is for Sm80 and Sm90 only")
class TestEVTLayoutSM90(EVTTestCaseBase):
@unittest.skipIf(device_cc() not in [80, 86, 89, 90], "This unittest is only supported on CC [80, 86, 89, 90]")
class TestEVTLayout(EVTTestCaseBase):
def test_permute_1(self):
"""
@@ -74,7 +74,7 @@ class TestEVTLayoutSM90(EVTTestCaseBase):
result_keys = ["D", "F"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() == 80, "This unittest is for cc = Sm90 only")
@unittest.skipIf(device_cc() != 90, "This unittest is for cc = Sm90 only")
def test_permute_2(self):
"""
Returning a tensor with shape [m, n]
@@ -99,7 +99,7 @@ class TestEVTLayoutSM90(EVTTestCaseBase):
result_keys = ["D", "F"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() == 80, "This unittest is for cc = Sm90 only")
@unittest.skipIf(device_cc() != 90, "This unittest is for cc = Sm90 only")
def test_permute_3(self):
"""
Returning a tensor with shape [m, n]
test/python/cutlass/evt/evt_load_sm80_90.py
+2 -2
View File
@@ -46,8 +46,8 @@ from utils.evt_testbed import EVTTestBed, EVTTestCaseBase
cutlass.set_log_level(logging.WARNING)
@unittest.skipIf(device_cc() not in [80, 90], "This unittest is for Sm80 and Sm90 only")
class TestEVTLoadSM90(EVTTestCaseBase):
@unittest.skipIf(device_cc() not in [80, 86, 89, 90], "This unittest is only supported on CC [80, 86, 89, 90]")
class TestEVTLoad(EVTTestCaseBase):
def test_tensor_load(self):
"""
test/python/cutlass/evt/evt_mixed_sm80_90.py
+6 -6
View File
@@ -47,8 +47,8 @@ from utils.evt_testbed import EVTTestBed, EVTTestCaseBase
cutlass.set_log_level(logging.WARNING)
@unittest.skipIf(device_cc() not in [80, 90], "This unittest is for Sm80 and Sm90 only")
class TestEVTMixedSM90(EVTTestCaseBase):
@unittest.skipIf(device_cc() not in [80, 86, 89, 90], "This unittest is only supported on CC [80, 86, 89, 90]")
class TestEVTMixed(EVTTestCaseBase):
def test_mixed_dag(self):
def evt_mixed_dag(accum, alpha, C, beta, aux, cbias, rbias):
F = alpha * accum + (beta * C + aux)
@@ -84,7 +84,7 @@ class TestEVTMixedSM90(EVTTestCaseBase):
result_keys = ["D", "F", "F_row_max", "E_col_max"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() != 80, "This unittest is for cc = Sm80 only")
@unittest.skipIf(device_cc() not in [80, 89], "This unittest is for cc 80 and 89 only")
def test_mixed_dag_float(self):
def evt_mixed_dag(accum, alpha, C, beta, aux, cbias, rbias):
F = alpha * accum + (beta * C + aux)
@@ -114,7 +114,7 @@ class TestEVTMixedSM90(EVTTestCaseBase):
result_keys = ["D", "F", "F_row_max", "E_col_max"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() != 80, "This unittest is for cc = Sm80 only")
@unittest.skipIf(device_cc() not in [80, 89], "This unittest is for cc 80 and 89 only")
def test_mixed_dag_stage2(self):
def evt_mixed_dag(accum, alpha, C, beta, aux, cbias, rbias):
F = alpha * accum + (beta * C + aux)
@@ -144,7 +144,7 @@ class TestEVTMixedSM90(EVTTestCaseBase):
result_keys = ["D", "F", "F_row_max", "E_col_max"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() != 80, "This unittest is for cc = Sm80 only")
@unittest.skipIf(device_cc() not in [80, 89], "This unittest is for cc 80 and 89 only")
def test_mixed_dag_partition_k(self):
def evt_mixed_dag(accum, alpha, C, beta, aux, cbias, rbias):
F = alpha * accum + (beta * C + aux)
@@ -179,7 +179,7 @@ class TestEVTMixedSM90(EVTTestCaseBase):
result_keys = ["D", "F", "F_row_max", "E_col_max"]
launcher.verify((m, n, k), input_keys, result_keys, l)
@unittest.skipIf(device_cc() != 80, "This unittest is for cc = Sm80 only")
@unittest.skipIf(device_cc() not in [80, 89], "This unittest is for cc 80 and 89 only")
def test_mixed_dag_stream_k(self):
def evt_mixed_dag(accum, alpha, C, beta, aux, cbias, rbias):
F = alpha * accum + (beta * C + aux)
test/python/cutlass/evt/evt_store_sm80_90.py
+2 -2
View File
@@ -46,8 +46,8 @@ from utils.evt_testbed import EVTTestBed, EVTTestCaseBase
cutlass.set_log_level(logging.WARNING)
@unittest.skipIf(device_cc() not in [80, 90], "This unittest is for Sm80 and Sm90 only")
class TestEVTStoreSM90(EVTTestCaseBase):
@unittest.skipIf(device_cc() not in [80, 86, 89, 90], "This unittest is only supported on CC [80, 86, 89, 90]")
class TestEVTStore(EVTTestCaseBase):
def test_aux_store(self):
"""
test/python/cutlass/gemm/gemm_f16_sm80.py
+4 -1
View File
@@ -46,8 +46,10 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 80
dtype = cutlass.DataType.f16
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF16Sm80(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -56,13 +58,14 @@ class GemmF16Sm80(unittest.TestCase):
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF16Sm80StreamK(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
"""
pass
add_test_specialized = partial(add_test_gemm, element=cutlass.DataType.f16, cc=cc, cluster_shape=[1, 1, 1])
add_test_specialized = partial(add_test_gemm, element=dtype, cc=cc, cluster_shape=[1, 1, 1])
# Tests using TensorOp
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
test/python/cutlass/gemm/gemm_f16_sm90.py
+3 -1
View File
@@ -46,8 +46,10 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 90
dtype = cutlass.DataType.f16
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM90 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF16Sm90(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -55,7 +57,7 @@ class GemmF16Sm90(unittest.TestCase):
pass
add_test_specialized = partial(add_test_gemm, cls=GemmF16Sm90, element=cutlass.DataType.f16,
add_test_specialized = partial(add_test_gemm, cls=GemmF16Sm90, element=dtype,
warp_count=None, compilation_modes=['nvcc'])
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
test/python/cutlass/gemm/gemm_f32_sm80.py
+25 -21
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 80
dtype = cutlass.DataType.f32
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF32Sm80(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -56,6 +59,7 @@ class GemmF32Sm80(unittest.TestCase):
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF32Sm80StreamK(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -63,37 +67,37 @@ class GemmF32Sm80StreamK(unittest.TestCase):
pass
add_test_specialized = partial(add_test_gemm, element=cutlass.DataType.f32, cc=cc, cluster_shape=[1, 1, 1])
add_test_specialized = partial(add_test_gemm, element=dtype, cc=cc, cluster_shape=[1, 1, 1])
# Tests using TensorOp
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NNN, alignments=[4, 4, 4], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NNT, alignments=[4, 4, 4], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[4, 4, 4], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[ 64, 128, 32], warp_count=[1, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[4, 4, 4], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[ 64, 64, 32], warp_count=[1, 1, 1], stages=4)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NNN, alignments=[4, 4, 4], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NNT, alignments=[4, 4, 4], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[4, 4, 4], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 128, 32], warp_count=[1, 2, 1], stages=3)
add_test_tensorop(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[4, 4, 4], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 64, 32], warp_count=[1, 1, 1], stages=4)
# Tests using SIMT
add_test_simt = partial(add_test_specialized, opclass=cutlass.OpcodeClass.Simt)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.TNN, alignments=[1, 1, 1], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[ 64, 128, 8], warp_count=[1, 2, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 64, 8], warp_count=[2, 1, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[ 64, 64, 8], warp_count=[1, 1, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NNT, alignments=[1, 1, 1], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.TNN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 128, 8], warp_count=[1, 2, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 64, 8], warp_count=[2, 1, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 64, 8], warp_count=[1, 1, 1], stages=2)
add_test_simt(cls=GemmF32Sm80, layouts=LayoutCombination.NNT, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
# Stream K tests
add_test_streamk = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp, swizzle=cutlass.swizzle.ThreadblockSwizzleStreamK)
add_test_streamk(cls=GemmF32Sm80StreamK, layouts=LayoutCombination.TTN, alignments=[4, 4, 4], element_output=cutlass.DataType.f32, element_C=cutlass.DataType.f32,
element_accumulator=cutlass.DataType.f32, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
add_test_streamk(cls=GemmF32Sm80StreamK, layouts=LayoutCombination.TTN, alignments=[4, 4, 4], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 32], warp_count=[2, 2, 1], stages=3)
if __name__ == '__main__':
test/python/cutlass/gemm/gemm_f64_sm80.py
+23 -19
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 80
dtype = cutlass.DataType.f64
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF64Sm80(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -56,6 +59,7 @@ class GemmF64Sm80(unittest.TestCase):
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF64Sm80StreamK(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -63,36 +67,36 @@ class GemmF64Sm80StreamK(unittest.TestCase):
pass
add_test_specialized = partial(add_test_gemm, element=cutlass.DataType.f64, cc=cc, cluster_shape=[1, 1, 1])
add_test_specialized = partial(add_test_gemm, element=dtype, cc=cc, cluster_shape=[1, 1, 1])
# Tests using TensorOp
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[128, 128, 16], warp_count=[4, 2, 1], stages=3)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[ 64, 64, 16], warp_count=[2, 2, 1], stages=4)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[ 32, 32, 16], warp_count=[2, 1, 1], stages=5)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 16], warp_count=[4, 2, 1], stages=3)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 64, 16], warp_count=[2, 2, 1], stages=4)
add_test_tensorop(cls=GemmF64Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 32, 32, 16], warp_count=[2, 1, 1], stages=5)
# Tests using SIMT
add_test_simt = partial(add_test_specialized, opclass=cutlass.OpcodeClass.Simt)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.TNN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[ 64, 128, 8], warp_count=[1, 2, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[128, 64, 8], warp_count=[2, 1, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[ 64, 64, 8], warp_count=[1, 1, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NNT, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NNN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.TNN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 128, 8], warp_count=[1, 2, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 64, 8], warp_count=[2, 1, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.TTN, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[ 64, 64, 8], warp_count=[1, 1, 1], stages=2)
add_test_simt(cls=GemmF64Sm80, layouts=LayoutCombination.NNT, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 8], warp_count=[2, 2, 1], stages=2)
# Stream K tests
add_test_streamk = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp, swizzle=cutlass.swizzle.ThreadblockSwizzleStreamK)
add_test_streamk(cls=GemmF64Sm80StreamK, layouts=LayoutCombination.NTT, alignments=[1, 1, 1], element_output=cutlass.DataType.f64, element_C=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, threadblock_shape=[128, 128, 16], warp_count=[4, 2, 1], stages=3)
add_test_streamk(cls=GemmF64Sm80StreamK, layouts=LayoutCombination.NTT, alignments=[1, 1, 1], element_output=dtype, element_C=dtype,
element_accumulator=dtype, threadblock_shape=[128, 128, 16], warp_count=[4, 2, 1], stages=3)
if __name__ == '__main__':
test/python/cutlass/gemm/gemm_f64_sm90.py
+4 -2
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 90
dtype = cutlass.DataType.f64
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM90 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF64Sm90(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -56,8 +59,7 @@ class GemmF64Sm90(unittest.TestCase):
add_test_specialized = partial(add_test_gemm, cls=GemmF64Sm90, alignments=[1, 1, 1], cluster_shape=[1, 1, 1],
element=cutlass.DataType.f64, element_output=cutlass.DataType.f64,
element_accumulator=cutlass.DataType.f64, compilation_modes=['nvcc'])
element=dtype, element_output=dtype, element_accumulator=dtype, compilation_modes=['nvcc'])
add_test_specialized(opclass=cutlass.OpcodeClass.TensorOp, layouts=LayoutCombination.NNT, threadblock_shape=[128, 128, 32], stages=3)
add_test_specialized(opclass=cutlass.OpcodeClass.TensorOp, layouts=LayoutCombination.TNN, threadblock_shape=[128, 128, 32], stages=3)
test/python/cutlass/gemm/gemm_f8_sm90.py
+112
View File
@@ -0,0 +1,112 @@
#################################################################################################
#
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#################################################################################################
"""
Low-level functionality tests for GEMM with S8 operands on SM90
"""
from functools import partial
import logging
import unittest
import cutlass
from cutlass.backend.utils.device import device_cc
from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 90
dtype = cutlass.DataType.e4m3
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM90 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF8E4M3Sm90(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
"""
pass
add_test_specialized = partial(add_test_gemm, cls=GemmF8E4M3Sm90, element=dtype, compilation_modes=['nvcc'])
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
# Test with 1x1x1 clusters
add_test_tensorop(layouts=LayoutCombination.TNT, alignments=[16, 16, 16], element_output=cutlass.DataType.e4m3,
element_accumulator=cutlass.DataType.f32, cluster_shape=[1, 1, 1], threadblock_shape=[128, 128, 128], stages=None)
# Tests with different cluster shapes
add_test_tensorop(layouts=LayoutCombination.TNT, alignments=[16, 16, 16], element_output=cutlass.DataType.e4m3,
element_accumulator=cutlass.DataType.f32, cluster_shape=[2, 2, 1], threadblock_shape=[128, 128, 128], stages=None)
add_test_tensorop(layouts=LayoutCombination.TNT, alignments=[16, 16, 16], element_output=cutlass.DataType.e4m3,
element_accumulator=cutlass.DataType.f32, cluster_shape=[1, 4, 1], threadblock_shape=[128, 128, 128], stages=None)
# Tests with warp-specialized ping-pong schedule
add_test_tensorop(layouts=LayoutCombination.TNT, alignments=[16, 16, 16], element_output=cutlass.DataType.e4m3,
element_accumulator=cutlass.DataType.f32, cluster_shape=[2, 1, 1], threadblock_shape=[128, 128, 128], stages=None,
kernel_schedule=cutlass.KernelScheduleType.TmaWarpSpecializedPingpong,
epilogue_schedule=cutlass.EpilogueScheduleType.TmaWarpSpecialized)
# Tests for SIMT
add_test_simt = partial(add_test_specialized, opclass=cutlass.OpcodeClass.Simt)
add_test_simt(layouts=LayoutCombination.TNN, alignments=[1, 1, 1], element_output=cutlass.DataType.e4m3,
element_accumulator=cutlass.DataType.f32, cluster_shape=[1, 1, 1], threadblock_shape=[64, 32, 8], stages=2)
#
# Add a test for E5M2
#
dtype = cutlass.DataType.e5m2
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM90 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmF8E5M2Sm90(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
"""
pass
add_test_specialized = partial(add_test_gemm, cls=GemmF8E5M2Sm90, element=dtype, compilation_modes=['nvcc'])
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
# Tests with 1x1x1 clusters
add_test_tensorop(layouts=LayoutCombination.TNN, alignments=[16, 16, 16], element_output=dtype,
element_accumulator=cutlass.DataType.f32, cluster_shape=[1, 1, 1], threadblock_shape=[128, 128, 128], stages=3)
if __name__ == '__main__':
unittest.main()
test/python/cutlass/gemm/gemm_mixed_sm80.py
+4 -1
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 80
dtype =cutlass.DataType.f16
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmMixedSm80(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -55,7 +58,7 @@ class GemmMixedSm80(unittest.TestCase):
pass
add_test_mixed = partial(add_test_gemm, cls=GemmMixedSm80, element=cutlass.DataType.f16, cc=cc, cluster_shape=[1, 1, 1],
add_test_mixed = partial(add_test_gemm, cls=GemmMixedSm80, element=dtype, cc=cc, cluster_shape=[1, 1, 1],
opclass=cutlass.OpcodeClass.TensorOp, threadblock_shape=[128, 128, 64],
warp_count=[2, 2, 1], stages=3, element_accumulator=cutlass.DataType.f32)
test/python/cutlass/gemm/gemm_s8_sm80.py
+5 -1
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 80
dtype = cutlass.DataType.s8
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmS8Sm80(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -56,6 +59,7 @@ class GemmS8Sm80(unittest.TestCase):
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM80 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmS8Sm80StreamK(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -63,7 +67,7 @@ class GemmS8Sm80StreamK(unittest.TestCase):
pass
add_test_specialized = partial(add_test_gemm, element=cutlass.DataType.s8, cc=cc, cluster_shape=[1, 1, 1])
add_test_specialized = partial(add_test_gemm, element=dtype, cc=cc, cluster_shape=[1, 1, 1])
# Tests using TensorOp
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
test/python/cutlass/gemm/gemm_s8_sm90.py
+4 -1
View File
@@ -46,8 +46,11 @@ from utils import LayoutCombination, add_test_gemm
cutlass.set_log_level(logging.WARNING)
cc = 90
dtype = cutlass.DataType.s8
@unittest.skipIf(device_cc() < cc, 'Device compute capability is insufficient for SM90 tests.')
@unittest.skipIf(cutlass.utils.datatypes.torch_type(dtype) is None, f'Version of torch installed does not contain a datatype match for {dtype}')
class GemmS8Sm90(unittest.TestCase):
"""
Wrapper class to which tests will be added dynamically in __main__
@@ -55,7 +58,7 @@ class GemmS8Sm90(unittest.TestCase):
pass
add_test_specialized = partial(add_test_gemm, cls=GemmS8Sm90, element=cutlass.DataType.s8, compilation_modes=['nvcc'])
add_test_specialized = partial(add_test_gemm, cls=GemmS8Sm90, element=dtype, compilation_modes=['nvcc'])
add_test_tensorop = partial(add_test_specialized, opclass=cutlass.OpcodeClass.TensorOp)
test/python/cutlass/gemm/gemm_testbed.py
+17 -2
View File
@@ -128,13 +128,22 @@ class GemmUniversalLauncher:
def uniform_init(self, shape, dtype, layout):
size = prod(shape)
if dtype.is_floating_point:
data = torch.ceil(torch.empty(size=(size,), dtype=dtype, device="cuda").uniform_(self.rand_min - 0.5, self.rand_max - 0.5))
# Initialize data in FP32 and call convert to the data type we desire.
# This is a workaround for the following error that occurs when attempting to
# call uniform_ on a tensor with torch.float8_e4m3fn data:
# RuntimeError: "check_uniform_bounds" not implemented for 'Float8_e4m3fn'
data = torch.ceil(
torch.empty(size=(size,), dtype=torch.float32, device="cuda").uniform_(
self.rand_min - 0.5, self.rand_max - 0.5)
).to(dtype)
else:
# PyTorch does not currently support integer-typed matrix multiplications on GPU.
# Fall back to CPU for integer type references.
data = torch.empty(size=(size,), dtype=dtype, device="cpu").random_(self.rand_min, self.rand_max + 1)
if dtype == torch.float64 or dtype == torch.float32:
is_fp8 = dtype == getattr(torch, "float8_e4m3fn", -1) or dtype == dtype == getattr(torch, "float8_e5m2", -1)
if dtype == torch.float64 or dtype == torch.float32 or is_fp8:
data = data.to("cpu")
data_ref = data.reshape(shape)
@@ -145,6 +154,12 @@ class GemmUniversalLauncher:
data_cutlass = data_ref.transpose(-1, -2).contiguous()
data_cutlass = data_cutlass.to("cuda")
# As of this writing, few operations in PyTorch are supported with FP8 data.
# Thus, we perform computation in FP32 for FP8 reference checks.
if is_fp8:
data_ref = data_ref.to(torch.float32)
return data_cutlass, data_ref
def reference(self, problem_size, tensor_A, tensor_B, tensor_C, alpha, beta):
test/unit/CMakeLists.txt
+4 -3
View File
@@ -65,10 +65,10 @@ function(cutlass_test_unit_add_executable NAME)
set(options WITHOUT_CUDA)
set(oneValueArgs)
set(multiValueArgs)
set(multiValueArgs TEST_SETS_SUPPORTED EXTRA_INCLUDE_DIRS)
cmake_parse_arguments(_ "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
cutlass_add_executable(${NAME} ${__UNPARSED_ARGUMENTS})
cutlass_add_executable(${NAME} ${__UNPARSED_ARGUMENTS} BATCH_SOURCES OFF)
target_compile_definitions(${NAME} PUBLIC CUTLASS_TARGET_NAME="${NAME}")
@@ -76,6 +76,7 @@ function(cutlass_test_unit_add_executable NAME)
${NAME}
PRIVATE
${CUTLASS_UNIT_TEST_COMMON_DIR}
${__EXTRA_INCLUDE_DIRS}
)
if (__WITHOUT_CUDA)
# Avoid CUDA dependencies for host-only unit tests that provide the
@@ -110,12 +111,12 @@ function(cutlass_test_unit_add_executable NAME)
cutlass_add_executable_tests(
${NAME_STEM} ${NAME}
TEST_SETS_SUPPORTED ${__TEST_SETS_SUPPORTED}
TEST_COMMAND_OPTIONS CUTLASS_TEST_UNIT_TEST_COMMAND_OPTIONS
${RESULT_CACHE_FILE_ARGS}
)
endfunction()
add_custom_target(cutlass_test_unit)
add_custom_target(test_unit)
test/unit/conv/device/cache_testbed_output.h → test/unit/conv/cache_testbed_output.h
+37 -7
View File
@@ -48,15 +48,15 @@
#include "cutlass/core_io.h"
#include "cutlass/util/tensor_view_io.h"
#include "thrust/universal_vector.h"
#ifndef CUTLASS_TEST_ENABLE_CACHED_RESULTS
#define CUTLASS_TEST_ENABLE_CACHED_RESULTS false
#endif
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace conv {
namespace device {
namespace test::conv::device {
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -325,7 +325,6 @@ inline std::ostream &EncodeProblemSize(
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Element>
inline std::string ElementTypeName() {
return std::string(typeid(Element).name());
@@ -452,6 +451,12 @@ inline std::string TensorTypeName() {
return ss.str();
}
template <typename Element>
inline std::string TensorTypeName() {
std::stringstream ss;
ss << ElementTypeName<Element>();
return ss.str();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Hash function on a byte array
@@ -511,6 +516,16 @@ uint32_t TensorHash(
return hash(view.data(), view.capacity() * cutlass::sizeof_bits<Element>::value / 8, crc);
}
template <typename Element>
uint32_t TensorHash(
thrust::universal_vector<Element>& tensor,
CRC32 const &hash = CRC32(),
uint32_t crc = uint32_t()
) {
return hash(tensor.data().get(), tensor.size() * cutlass::sizeof_bits<Element>::value / 8, crc);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <
@@ -533,6 +548,23 @@ inline std::ostream &EncodeTypes(
return out;
}
template <
typename ElementA,
typename ElementB,
typename ElementC,
typename ElementD
>
inline std::ostream &EncodeTypes(
std::ostream &out
) {
out << TensorTypeName<ElementA>() << "_"
<< TensorTypeName<ElementB>() << "_"
<< TensorTypeName<ElementC>() << "_"
<< ElementTypeName<ElementD>();
return out;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <
@@ -790,8 +822,6 @@ inline CachedTestKey CreateCachedConv3dTestKey(
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // nammespace conv
} // namespace test
} // namespace test::conv::device
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/conv/device/conv2d_testbed.h
+1 -1
View File
@@ -55,7 +55,7 @@
#include "cutlass/core_io.h"
#include "cutlass/util/tensor_view_io.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
namespace test {
namespace conv {
test/unit/conv/device/conv2d_testbed_interleaved.h
+1 -1
View File
@@ -56,7 +56,7 @@
#include "cutlass/core_io.h"
#include "cutlass/util/tensor_view_io.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
namespace test {
namespace conv {
test/unit/conv/device/conv2d_with_broadcast_testbed.h
+1 -1
View File
@@ -59,7 +59,7 @@
#include "cutlass/core_io.h"
#include "cutlass/util/tensor_view_io.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
namespace test {
namespace conv {
test/unit/conv/device/conv2d_with_reduction_testbed.h
+1 -1
View File
@@ -56,7 +56,7 @@
#include "cutlass/core_io.h"
#include "cutlass/util/tensor_view_io.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
namespace test {
namespace conv {
test/unit/conv/device/conv3d_testbed.h
+1 -1
View File
@@ -55,7 +55,7 @@
#include "conv3d_problems.h"
#include "cutlass/core_io.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
namespace test {
namespace conv {
test/unit/conv/device/depthwise_conv2d_direct_conv_testbed.h
+4 -4
View File
@@ -36,7 +36,7 @@
#include <fstream>
#include "../../common/cutlass_unit_test.h"
#include "cache_testbed_output.h"
#include "../cache_testbed_output.h"
#include "conv2d_problems.h"
#include "cutlass/conv/device/direct_convolution.h"
@@ -466,8 +466,8 @@ bool TestSpecificDepthwiseDirectConv2d(const Conv2dProblemVector &problem_sizes)
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace conv
} // namespace test
} // namespace device
} // namespace conv
} // namespace test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/core/CMakeLists.txt
+7
View File
@@ -64,3 +64,10 @@ add_executable(
cpp11.cu
)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
target_compile_options(
cutlass_test_unit_core_cpp11
PRIVATE
$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler -Werror>
)
endif()
test/unit/core/cpp11.cu
+2 -1
View File
@@ -42,6 +42,7 @@
#include <cutlass/cutlass.h>
#include <cutlass/complex.h>
#include <cutlass/coord.h>
#include <cutlass/core_io.h>
#include <cutlass/array.h>
@@ -51,12 +52,12 @@
#include <cutlass/half.h>
#include <cutlass/integer_subbyte.h>
#include <cutlass/kernel_hardware_info.h>
#include <cutlass/numeric_conversion.h>
#include <cutlass/numeric_size.h>
#include <cutlass/numeric_types.h>
#include <cutlass/tfloat32.h>
#include <cutlass/workspace.h>
#include <cutlass/subbyte_reference.h>
#include <cutlass/conv/convolution.h>
#include <cutlass/conv/conv2d_problem_size.h>
test/unit/core/fast_numeric_conversion.cu
+14
View File
@@ -147,6 +147,20 @@ TEST(FastNumericConversion, s32_to_f32) {
test::core::kernel::run_test_integer_range_limited<Destination, Source, kN>();
}
TEST(FastNumericConversion, s8_to_f32_array) {
int const kN = 256;
using Source = int8_t;
using Destination = float;
test::core::kernel::run_test_integer_range_all<Destination, Source, kN>();
}
TEST(FastNumericConversion, u8_to_f32_array) {
int const kN = 256;
using Source = uint8_t;
using Destination = float;
test::core::kernel::run_test_integer_range_all<Destination, Source, kN>();
}
TEST(FastNumericConversion, s8_to_f16_array) {
int const kN = 256;
using Source = int8_t;
test/unit/core/numeric_conversion.cu
+159 -3
View File
@@ -60,8 +60,8 @@ __global__ void convert(
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Destination, typename Source, int Count, int Range = 4>
void run_test(const char dest_name[], const char source_name[]) {
template <typename Destination, typename Source, int Count>
void run_test(const char dest_name[], const char source_name[], const int range = 4, const int offset = 0) {
const int kN = Count;
dim3 grid(1, 1);
@@ -73,7 +73,7 @@ void run_test(const char dest_name[], const char source_name[]) {
auto destination_ref = destination.host_ref();
for (int i = 0; i < kN; ++i) {
source_ref.at({0, i}) = Source(i % Range);
source_ref.at({0, i}) = Source(i % range + offset);
}
source.sync_device();
@@ -509,4 +509,160 @@ TEST(NumericConversion, int_to_fe4m3_t_array_32) {
test::core::kernel::run_test<Destination, Source, kN>(dest_name, source_name);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
struct GetName {
static constexpr char name[] = "UNSUPPORTED";
};
template <>
struct GetName<cutlass::int4b_t> {
static constexpr char name[] = "int4b_t";
};
template <>
struct GetName<uint8_t> {
static constexpr char name[] = "uint8_t";
};
template <>
struct GetName<int8_t> {
static constexpr char name[] = "int8_t";
};
template <>
struct GetName<cutlass::float_e4m3_t> {
static constexpr char name[] = "float_e4m3_t";
};
template <>
struct GetName<cutlass::half_t> {
static constexpr char name[] = "half_t";
};
template <>
struct GetName<cutlass::bfloat16_t> {
static constexpr char name[] = "bfloat16_t";
};
template <>
struct GetName<float> {
static constexpr char name[] = "float";
};
template <typename Result_, typename Source_>
struct ResultSourcePair {
using Result = Result_;
using Source = Source_;
};
template <typename ResultSourcePair>
class VectorArrayConverterTest : public testing::Test {
public:
using Result = typename ResultSourcePair::Result;
using Source = typename ResultSourcePair::Source;
template <int N>
static void emit_test() {
const int range = 1 << cutlass::sizeof_bits<Source>::value;
const int offset = cutlass::platform::numeric_limits<Source>::lowest();
test::core::kernel::run_test<Result, Source, N>(GetName<Result>::name, GetName<Source>::name, range, offset);
}
};
using VectorConvertTypes = ::testing::Types<
ResultSourcePair<float, int8_t>,
ResultSourcePair<float, uint8_t>,
ResultSourcePair<cutlass::half_t, int8_t>,
ResultSourcePair<cutlass::half_t, uint8_t>,
ResultSourcePair<cutlass::bfloat16_t, uint8_t>,
ResultSourcePair<cutlass::bfloat16_t, int8_t>,
ResultSourcePair<cutlass::float_e4m3_t, cutlass::int4b_t>,
ResultSourcePair<cutlass::half_t, cutlass::int4b_t>,
ResultSourcePair<cutlass::bfloat16_t, cutlass::int4b_t>,
ResultSourcePair<float, cutlass::int4b_t>
>;
TYPED_TEST_SUITE(VectorArrayConverterTest, VectorConvertTypes);
TYPED_TEST(VectorArrayConverterTest, array_1) {
TestFixture::template emit_test<1>();
}
TYPED_TEST(VectorArrayConverterTest, array_2) {
TestFixture::template emit_test<2>();
}
TYPED_TEST(VectorArrayConverterTest, array_3) {
TestFixture::template emit_test<3>();
}
TYPED_TEST(VectorArrayConverterTest, array_4) {
TestFixture::template emit_test<4>();
}
TYPED_TEST(VectorArrayConverterTest, array_5) {
TestFixture::template emit_test<5>();
}
TYPED_TEST(VectorArrayConverterTest, array_8) {
TestFixture::template emit_test<8>();
}
TYPED_TEST(VectorArrayConverterTest, array_10) {
// N > 8 and N is not a multiple of 4
TestFixture::template emit_test<10>();
}
TYPED_TEST(VectorArrayConverterTest, array_12) {
// N > 8 and N is a multiple of 4
TestFixture::template emit_test<12>();
}
TYPED_TEST(VectorArrayConverterTest, array_16) {
// N > 8 and N is a multiple of 8
TestFixture::template emit_test<16>();
}
TYPED_TEST(VectorArrayConverterTest, array_17) {
// N > 8 and N is not a multiple of 8
TestFixture::template emit_test<17>();
}
TYPED_TEST(VectorArrayConverterTest, array_27) {
// Test entire conversion range with residue (for int4)
TestFixture::template emit_test<27>();
}
TYPED_TEST(VectorArrayConverterTest, array_31) {
// Force use of converters for 16, 8, 4, 2 and scalar
// if max width is 16
TestFixture::template emit_test<31>();
}
TYPED_TEST(VectorArrayConverterTest, array_63) {
// Force use of converters for 32, 16, 8, 4, 2 and scalar
// if max width is 32
TestFixture::template emit_test<63>();
}
TYPED_TEST(VectorArrayConverterTest, array_256) {
// Test entire conversion range (for int8)
TestFixture::template emit_test<256>();
}
TYPED_TEST(VectorArrayConverterTest, array_259) {
// Force use of 4, 2 and scalar converter (if max width is 4)
TestFixture::template emit_test<259>();
}
TYPED_TEST(VectorArrayConverterTest, array_263) {
// Force use of 8, 4, 2 and scalar converter (if max width is 8)
TestFixture::template emit_test<263>();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/cute/core/CMakeLists.txt
+2
View File
@@ -42,10 +42,12 @@ cutlass_test_unit_add_executable(
inverse_right.cpp
logical_divide.cpp
logical_product.cpp
math.cpp
mixedbits.cpp
nullspace.cpp
pointer.cpp
reverse.cpp
transform.cpp
tuple.cpp
int_tuple.cpp
)
test/unit/cute/core/int_tuple.cpp
+131
View File
@@ -0,0 +1,131 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cute/layout.hpp>
TEST(CuTe_core, WeaklyCongruent)
{
using namespace cute;
auto a = _1{};
auto b = _2{};
EXPECT_TRUE (weakly_congruent(a, a));
EXPECT_TRUE (weakly_congruent(b, b));
EXPECT_TRUE (weakly_congruent(a, b));
auto a0 = Shape<_1>{};
auto b0 = Shape<_2>{};
EXPECT_TRUE (weakly_congruent(a , a0));
EXPECT_TRUE (weakly_congruent(b , b0));
EXPECT_TRUE (weakly_congruent(a , b0));
EXPECT_TRUE (weakly_congruent(b , a0));
EXPECT_FALSE(weakly_congruent(a0, a ));
EXPECT_FALSE(weakly_congruent(b0, b ));
EXPECT_FALSE(weakly_congruent(a0, b ));
EXPECT_FALSE(weakly_congruent(b0, a ));
EXPECT_TRUE (weakly_congruent(a0, a0));
EXPECT_TRUE (weakly_congruent(b0, b0));
EXPECT_TRUE (weakly_congruent(a0, b0));
auto a1 = Shape<_1, _1>{};
EXPECT_TRUE (weakly_congruent(a , a1));
EXPECT_FALSE(weakly_congruent(a0, a1));
EXPECT_TRUE (weakly_congruent(a1, a1));
auto a2 = Shape<_1, Shape<_1,_1>>{};
EXPECT_TRUE (weakly_congruent(a , a2));
EXPECT_FALSE(weakly_congruent(a0, a2));
EXPECT_TRUE (weakly_congruent(a1, a2));
auto b1 = Shape<_2, _2>{};
EXPECT_TRUE (weakly_congruent(b , b1));
EXPECT_FALSE(weakly_congruent(b0, b1));
EXPECT_TRUE (weakly_congruent(a1, b1));
auto b2 = Shape<_2, Shape<_2,_2>>{};
EXPECT_FALSE(weakly_congruent(a2, b0));
EXPECT_FALSE(weakly_congruent(a2, a1));
EXPECT_TRUE (weakly_congruent(a2, b2));
auto b3 = Shape<Shape<_2,_2>, Shape<_2,_2>>{};
EXPECT_FALSE(weakly_congruent(a0, b3));
EXPECT_TRUE (weakly_congruent(a1, b3));
EXPECT_TRUE (weakly_congruent(a2, b3));
}
TEST(CuTe_core, WeaklyCompatible)
{
using namespace cute;
auto a = _16{};
auto b = _12{};
auto c = _8{};
EXPECT_TRUE (weakly_compatible(a, a));
EXPECT_TRUE (weakly_compatible(b, b));
EXPECT_TRUE (weakly_compatible(c, c));
EXPECT_FALSE(weakly_compatible(a, b));
EXPECT_FALSE(weakly_compatible(a, c));
EXPECT_TRUE (weakly_compatible(c, a));
auto a0 = Shape<_16>{};
EXPECT_TRUE (weakly_compatible(a0, a0));
EXPECT_TRUE (weakly_compatible(a , a0));
EXPECT_FALSE(weakly_compatible(a0, a ));
EXPECT_TRUE (weakly_compatible(c , a0));
EXPECT_FALSE(weakly_compatible(a0, c ));
EXPECT_FALSE(weakly_compatible(b , a0));
EXPECT_FALSE(weakly_compatible(a0, b ));
auto a1 = Shape<_2,_8>{};
EXPECT_TRUE (weakly_compatible(a1, a1));
EXPECT_TRUE (weakly_compatible(a , a1));
EXPECT_FALSE(weakly_compatible(a0, a1));
EXPECT_FALSE(weakly_compatible(a1, a0));
EXPECT_TRUE (weakly_compatible(a1, Shape<_2,Shape<_2,_4>>{}));
auto a2 = Shape<Shape<_2,_8>>{};
EXPECT_TRUE (weakly_compatible(a2, a2));
EXPECT_TRUE (weakly_compatible(a , a2));
EXPECT_TRUE (weakly_compatible(c , a2));
EXPECT_TRUE (weakly_compatible(a0, a2));
EXPECT_FALSE(weakly_compatible(a2, a0));
auto a3 = Shape<Shape<_2,Shape<_4,_2>>>{};
EXPECT_TRUE (weakly_compatible(a3, a3));
EXPECT_TRUE (weakly_compatible(a , a3));
EXPECT_TRUE (weakly_compatible(c , a3));
EXPECT_TRUE (weakly_compatible(a0, a3));
EXPECT_FALSE(weakly_compatible(a3, a0));
EXPECT_TRUE (weakly_compatible(a2, a3));
EXPECT_FALSE(weakly_compatible(a3, a2));
}
test/unit/cute/core/math.cpp
+125
View File
@@ -0,0 +1,125 @@
/***************************************************************************************************
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include <cutlass/trace.h>
#include <cute/numeric/integral_constant.hpp>
#include <cute/numeric/math.hpp>
#include <cute/util/type_traits.hpp>
// If cute::gcd returns auto instead of common_type_t<T, U>,
// then GCC 7.5 reports the following error;
//
// ... /include/cute/numeric/math.hpp:103:26: error:
// inconsistent deduction for auto return type: ‘int’ and then ‘bool’
// if (u == 0) { return t; }
// ^
// Note that common_type_t<C<42>, C<1>>::value_type might still be bool.
TEST(CuTe_core, gcd_returns_common_type)
{
using cute::C;
constexpr auto fifteen = C<3 * 5>{};
static_assert(cute::is_same_v<decltype(fifteen)::value_type, int>);
static_assert(int(fifteen) == 15);
constexpr auto forty_two = C<2 * 3 * 7>{};
static_assert(cute::is_same_v<decltype(forty_two)::value_type, int>);
static_assert(int(forty_two) == 42);
// C<1>::value_type (as well as C<0>::value_type) may be bool.
constexpr auto one = C<1>{};
// Both inputs have value_type int.
{
constexpr auto result = cute::gcd(fifteen, forty_two);
static_assert(cute::is_same_v<decltype(result)::value_type, int>);
static_assert(int(result) == 3);
}
// One input has value_type int, and the other may have value_type bool.
{
constexpr auto result = cute::gcd(one, forty_two);
static_assert(int(result) == 1);
}
{
constexpr auto result = cute::gcd(forty_two, one);
static_assert(int(result) == 1);
}
// Both inputs may have value_type bool.
{
constexpr auto result = cute::gcd(one, one);
static_assert(int(result) == 1);
}
}
TEST(CuTe_core, lcm_returns_common_type)
{
using cute::C;
constexpr auto six = C<2 * 3>{};
static_assert(cute::is_same_v<decltype(six)::value_type, int>);
static_assert(int(six) == 6);
constexpr auto fifteen = C<3 * 5>{};
static_assert(cute::is_same_v<decltype(fifteen)::value_type, int>);
static_assert(int(fifteen) == 15);
// C<1>::value_type (as well as C<0>::value_type) may be bool.
constexpr auto one = C<1>{};
// Both inputs have value_type int.
{
constexpr auto result = cute::lcm(six, fifteen);
static_assert(cute::is_same_v<decltype(result)::value_type, int>);
static_assert(int(result) == 30);
}
// One input has value_type int, and the other may have value_type bool.
{
constexpr auto result = cute::lcm(one, six);
static_assert(cute::is_same_v<decltype(result)::value_type, int>);
static_assert(int(result) == 6);
}
{
constexpr auto result = cute::lcm(six, one);
static_assert(cute::is_same_v<decltype(result)::value_type, int>);
static_assert(int(result) == 6);
}
// Both inputs may have value_type bool.
{
constexpr auto result = cute::lcm(one, one);
static_assert(int(result) == 1);
}
}
test/unit/cute/hopper/tma_load_testbed.hpp
+2
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
#pragma once
#include "cutlass_unit_test.h"
#include <iostream>
test/unit/cute/hopper/tma_store_testbed.hpp
+2
View File
@@ -29,6 +29,8 @@
*
**************************************************************************************************/
#pragma once
#include "cutlass_unit_test.h"
#include <iostream>
test/unit/gemm/device/default_gemm_configuration.hpp
+22 -20
View File
@@ -169,7 +169,7 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_16x8x16_F32F16F16F32_TN>,
Layout<Shape<_2,_2,_1>>, // 2x2x1 thread group
Layout<Shape<_1,_2,_1>>>; // 1x2x1 value group for 16x16x16 MMA and LDSM
Tile<_32,_32,_16>>; // 32x32x16 MMA for LDSM, 1x2x1 value group
// A
static constexpr int kAlignmentA = 8;
@@ -301,7 +301,7 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_16x8x8_F32TF32TF32F32_TN>,
Layout<Shape<_2,_2,_1>, Stride<_2, _1, _1>>, // 2x2x1 thread group
Layout<Shape<_1,_2,_1>>>; // 1x2x1 value group for 16x16x8 and LDSM
Tile<_32,_32,_8>>; // 32x32x8 MMA for LDSM, 1x2x1 value group
// A
static constexpr int kAlignmentA = 4;
@@ -352,7 +352,7 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_16x8x32_S32S8S8S32_TN>,
Layout<Shape<_2,_2,_1>>, // 2x2x1 thread group
Layout<Shape<_1,_2,_1>>>; // 1x2x1 value group for 16x16x32 and LDSM
Tile<_32,_32,_32>>; // 16x16x32 MMA for LDSM, 1x2x1 value group
// A (M,K) K-major
using SmemLayoutAtomA = decltype(
@@ -798,9 +798,9 @@ struct DefaultGemmConfigurationToCutlass3Types<
using DispatchPolicy = MainloopSm80CpAsync<3>;
using TiledMma = TiledMMA<
MMA_Atom<UniversalFMA<ElementAccumulator, ElementA, ElementB, ElementC>>,
Layout<Shape<_16, _16, _1>>,
Layout<Shape< _2, _2, _1>>,
Tile<Layout<_2,_16>,Layout<_2,_16>,Underscore>>;
Layout<Shape<_16, _16, _1>>, // 16x16x1 thread group
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>, // 32x32x1 MMA with perm for load vectorization
Layout<Shape<_16,_2>,Stride<_2,_1>>,Underscore>>;
// A (M,K) M-major
using SmemLayoutAtomA = Layout<Shape<_128,_16>>;
@@ -920,9 +920,8 @@ struct DefaultGemmConfigurationToCutlass3Types<
using DispatchPolicy = MainloopSm80CpAsync<3>;
using TiledMma = TiledMMA<
MMA_Atom<UniversalFMA<ElementAccumulator, ElementA, ElementB, ElementC>>,
Layout<Shape<_16, _16, _1>>,
Layout<Shape< _2, _1, _1>>,
Tile<Layout<_2,_16>,Underscore,Underscore>>;
Layout<Shape<_16, _16, _1>>, // 16x16x1 thread group
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>,Underscore,Underscore>>; // 32x16x1 MMA with perm for load vectorization
// A (M,K) M-major
using SmemLayoutAtomA = Layout<Shape<_128,_16>>;
@@ -982,9 +981,8 @@ struct DefaultGemmConfigurationToCutlass3Types<
using DispatchPolicy = MainloopSm80CpAsync<3>;
using TiledMma = TiledMMA<
MMA_Atom<UniversalFMA<ElementAccumulator, ElementA, ElementB, ElementC>>,
Layout<Shape<_16, _16, _1>>,
Layout<Shape< _1, _2, _1>>,
Tile<Underscore,Layout<_2,_16>,Underscore>>;
Layout<Shape<_16, _16, _1>>, // 16x16x1 thread group
Tile<Underscore,Layout<Shape<_16,_2>,Stride<_2,_1>>,Underscore>>; // 16x32x1 MMA with perm for load vectorization
// A (M,K) K-major
using SmemLayoutAtomA = Layout<Shape <_128, _16>,
@@ -1041,8 +1039,9 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>, // Atom
Layout<Shape<_2,_2,_1>>, // Atom layout
Layout<Shape<_2,_2,_1>>, // Val layout
Tile<Layout<_2,_16>,Layout<_2,_16>,Underscore>>; // Mode permutations
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>, // 32x32x4 MMA with perm for load vectorization
Layout<Shape<_16,_2>,Stride<_2,_1>>,
Underscore>>;
// A (M,K) K-Major
using SmemLayoutAtomA = decltype(
@@ -1119,8 +1118,9 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>, // Atom
Layout<Shape<_2,_2,_1>>, // Atom layout
Layout<Shape<_2,_2,_1>>, // Val layout
Tile<Layout<_2,_16>,Layout<_2,_16>,Underscore>>; // Mode permutations
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>, // 32x32x4 MMA with perm for load vectorization
Layout<Shape<_16,_2>,Stride<_2,_1>>,
Underscore>>;
// A (M,K) M-Major
using SmemLayoutAtomA = decltype(
@@ -1183,8 +1183,9 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>, // Atom
Layout<Shape<_2,_2,_1>>, // Atom layout
Layout<Shape<_2,_2,_1>>, // Val layout
Tile<Layout<_2,_16>,Layout<_2,_16>,Underscore>>; // Mode permutations
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>, // 32x32x4 MMA with perm for load vectorization
Layout<Shape<_16,_2>,Stride<_2,_1>>,
Underscore>>;
// A (M,K) M-Major
using SmemLayoutAtomA = decltype(
@@ -1247,8 +1248,9 @@ struct DefaultGemmConfigurationToCutlass3Types<
using TiledMma = TiledMMA<
MMA_Atom<SM80_8x8x4_F64F64F64F64_TN>, // Atom
Layout<Shape<_2,_2,_1>>, // Atom layout
Layout<Shape<_2,_2,_1>>, // Val layout
Tile<Layout<_2,_16>,Layout<_2,_16>,Underscore>>; // Mode permutations
Tile<Layout<Shape<_16,_2>,Stride<_2,_1>>, // 32x32x4 MMA with perm for load vectorization
Layout<Shape<_16,_2>,Stride<_2,_1>>,
Underscore>>;
// A (M,K) K-Major
using SmemLayoutAtomA = decltype(
test/unit/gemm/device/gemm_testbed_3x.hpp
+106 -53
View File
@@ -58,6 +58,7 @@
#include "cutlass/fast_math.h"
#include "cutlass/platform/platform.h"
#include "cutlass/epilogue/fusion/operations.hpp"
#include "cutlass/gemm/kernel/tile_scheduler_params.h"
#include "cute/int_tuple.hpp"
#include "cute/layout.hpp"
@@ -192,6 +193,7 @@ struct TestbedImpl {
using ActivationFunctor = ActivationFunctor_<ElementCompute>;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
static_assert(cute::rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
static_assert(cute::rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
@@ -248,6 +250,10 @@ struct TestbedImpl {
// Used to force multi-wave tests for persistent kernel schedules
constexpr static int MaxSmCount = 16;
cutlass::ComplexTransform TransformA = Gemm::kTransformA;
cutlass::ComplexTransform TransformB = Gemm::kTransformB;
//
// Methods
//
@@ -462,7 +468,7 @@ struct TestbedImpl {
auto Vbeta = cute::make_tensor(static_cast<ElementCompute*>(nullptr),
cute::make_layout(cute::make_shape(M, cute::_1{})));
cutlass::reference::host::GettMainloopParams<ElementAccumulator, decltype(A), decltype(B)> mainloop_params{A, B};
cutlass::reference::host::GettMainloopParams<ElementAccumulator, decltype(A), decltype(B)> mainloop_params{A, B, TransformA, TransformB};
cutlass::reference::host::GettEpilogueParams<
ElementScalar,
@@ -523,6 +529,9 @@ struct TestbedImpl {
Gemm& gemm_op,
typename Gemm::Arguments& arguments,
cutlass::device_memory::allocation<uint8_t>& workspace) {
int M = cute::size<0>(problem_size);
int N = cute::size<1>(problem_size);
int K = cute::size<2>(problem_size);
int L = 1;
if constexpr(cute::rank(ProblemShapeType{}) == 4) {
L = cute::size<3>(problem_size);
@@ -561,7 +570,8 @@ struct TestbedImpl {
detail::Iterations iterations = detail::Iterations{},
RasterOrderOptions raster_order = RasterOrderOptions::Heuristic,
detail::MaxSwizzleSize max_swizzle = detail::MaxSwizzleSize{},
detail::Splits splits = detail::Splits{})
detail::Splits splits = detail::Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic)
{
// Fail test if insufficient CUDA device
if (!sufficient()) {
@@ -586,14 +596,6 @@ struct TestbedImpl {
hw_info.sm_count = this->sm_count;
}
typename Gemm::GemmKernel::TileScheduler::Arguments scheduler_args;
if constexpr (std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
scheduler_args = { static_cast<int>(splits), static_cast<int>(max_swizzle), raster_order };
}
else {
scheduler_args = { static_cast<int>(max_swizzle), raster_order };
}
// DefaultEpilogue
auto arguments = typename Gemm::Arguments {
cutlass::gemm::GemmUniversalMode::kGemm,
@@ -606,10 +608,20 @@ struct TestbedImpl {
{alpha, beta},
tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d
},
hw_info,
scheduler_args
hw_info
};
if constexpr (std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
arguments.scheduler.splits = static_cast<int>(splits);
arguments.scheduler.max_swizzle_size = static_cast<int>(max_swizzle);
arguments.scheduler.raster_order = raster_order;
arguments.scheduler.decomposition_mode = decomposition_mode;
} else {
arguments.scheduler.max_swizzle_size = static_cast<int>(max_swizzle);
arguments.scheduler.raster_order = raster_order;
}
Gemm gemm_op;
size_t workspace_size = Gemm::get_workspace_size(arguments);
@@ -683,6 +695,7 @@ struct Testbed3x {
using LayoutTagD = typename TestBedImpl::LayoutTagD;
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
// Detail Implementation
TestBedImpl impl_;
@@ -723,11 +736,12 @@ struct Testbed3x {
RasterOrderOptions raster_order = RasterOrderOptions::Heuristic,
detail::MaxSwizzleSize max_swizzle = detail::MaxSwizzleSize{},
detail::Splits splits = detail::Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic,
bool profiling = false,
detail::Iterations iterations = detail::Iterations{})
{
return impl_.run(
problem_size, alpha, beta, profiling, iterations, raster_order, max_swizzle, splits
problem_size, alpha, beta, profiling, iterations, raster_order, max_swizzle, splits, decomposition_mode
);
}
};
@@ -768,6 +782,7 @@ struct Testbed3xFusionOperation {
static_assert(cute::is_base_of_v<cutlass::epilogue::fusion::FusionOperation, FusionOp>);
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
// fusion types are potentially void if the fusion is not supported
// helper so we don't try to construct HostTensor with void type
@@ -818,6 +833,7 @@ struct Testbed3xFusionOperation {
cutlass::HostTensor<ElementAmax, LayoutTagScalar> abs_max_D;
cutlass::HostTensor<ElementAux , LayoutTagAux > tensor_Aux;
cutlass::gemm::TagToStrideC_t< LayoutTagAux > stride_Aux;
// References
cutlass::HostTensor<ElementBias, LayoutTagVector> reference_dbias;
cutlass::HostTensor<ElementAux , LayoutTagAux > reference_Aux;
@@ -977,7 +993,6 @@ struct Testbed3xFusionOperation {
cutlass::reference::host::TensorFill(reference_abs_max_Aux.host_view(), ElementAmax(0));
}
}
}
template <
@@ -1219,6 +1234,7 @@ struct Testbed3xFusionOperation {
RasterOrderOptions raster_order = RasterOrderOptions::Heuristic,
detail::MaxSwizzleSize max_swizzle = detail::MaxSwizzleSize{},
detail::Splits splits = detail::Splits{},
DecompositionMode decomposition_mode = DecompositionMode::Heuristic,
bool profiling = false,
detail::Iterations iterations = detail::Iterations{})
{
@@ -1234,7 +1250,7 @@ struct Testbed3xFusionOperation {
typename Gemm::Arguments arguments;
cutlass::KernelHardwareInfo hw_info;
cudaDeviceProp prop;
hw_info.device_id = 0;
if (not profiling) {
impl_.sm_count = std::min(impl_.MaxSmCount, cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id));
@@ -1251,11 +1267,6 @@ struct Testbed3xFusionOperation {
/// A/B/C/D Tensor
initialize(problem_size, alpha_, beta_);
typename Gemm::GemmKernel::TileScheduler::Arguments scheduler_args;
if constexpr (std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
scheduler_args = { static_cast<int>(splits) };
}
arguments = typename Gemm::Arguments{
cutlass::gemm::GemmUniversalMode::kGemm,
problem_size,
@@ -1270,10 +1281,19 @@ struct Testbed3xFusionOperation {
impl_.tensor_D.device_data(),
impl_.stride_d
}, // Epilogue arguments end
hw_info,
scheduler_args
hw_info
};
if constexpr (std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
arguments.scheduler.splits = static_cast<int>(splits);
arguments.scheduler.max_swizzle_size = static_cast<int>(max_swizzle);
arguments.scheduler.raster_order = raster_order;
arguments.scheduler.decomposition_mode = decomposition_mode;
} else {
arguments.scheduler.max_swizzle_size = static_cast<int>(max_swizzle);
arguments.scheduler.raster_order = raster_order;
}
auto coord_0 = cutlass::make_Coord(0);
if constexpr (IsLegacy) {
arguments.epilogue.thread = {
@@ -1313,12 +1333,18 @@ struct Testbed3xFusionOperation {
}
// example of how to set kernel activation arguments
// see ActivationFunctor::Arguments in activation.h for definition
// if Arguments doesn't exist then fusion_args.activation is empty
if constexpr (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::ScaledGELU_taylor<ElementCompute>>) {
// see ActivationFunctor::Arguments in activation.h for definition
// if Arguments doesn't exist then fusion_args.activation is empty
fusion_args.activation.scale = ElementCompute(1);
}
// Treat Clamp as ReLU
if constexpr (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::Clamp<ElementCompute>>) {
fusion_args.activation.lower_bound = 0;
fusion_args.activation.upper_bound = std::numeric_limits<ElementCompute>::max();
}
if constexpr (IsAbsMaxEnabledD) {
fusion_args.amax_D_ptr = abs_max_D.device_data();
}
@@ -1381,7 +1407,6 @@ struct Testbed3xFusionOperation {
std::cout << "Error : Failed : with alpha: " << float(alpha_) << ", beta: " << float(beta_)
<< "\n";
}
return passed;
}
}
@@ -1413,13 +1438,21 @@ bool TestAll(double alpha = 1.0, double beta = 0.0, Testbed testbed = {}) {
std::vector<int> problem_size_k = {max_alignment, TileShapeK * (Stages + 1) - max_alignment};
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
std::vector<DecompositionMode> decomposition_modes = {DecompositionMode::Heuristic};
std::vector<int> problem_splits = {1};
if constexpr (std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
static constexpr bool UsesStreamKScheduler = std::is_same_v<typename Gemm::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>;
if constexpr (UsesStreamKScheduler) {
problem_splits.push_back(2);
problem_splits.push_back(3);
// As many splits as there are maximum k tiles
problem_splits.push_back(Stages + 1);
decomposition_modes.push_back(DecompositionMode::DataParallel);
decomposition_modes.push_back(DecompositionMode::SplitK);
decomposition_modes.push_back(DecompositionMode::StreamK);
// Use larger K sizes for stream-K tests
static constexpr int min_tiles_per_sk_unit = cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::min_iters_per_sk_unit_;
problem_size_k = {TileShapeK * min_tiles_per_sk_unit, TileShapeK * 3 * min_tiles_per_sk_unit - max_alignment};
}
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
@@ -1433,33 +1466,53 @@ bool TestAll(double alpha = 1.0, double beta = 0.0, Testbed testbed = {}) {
for (int k : problem_size_k) {
for (auto raster_order : raster_orders) {
for (int max_swizzle_size : max_swizzle_sizes) {
for (int splits : problem_splits) {
ProblemShapeType problem_size;
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
problem_size = ProblemShapeType{m, n, k, /* l */ 1};
}
else {
problem_size = ProblemShapeType{m, n, k};
}
for (DecompositionMode decomp_mode : decomposition_modes) {
passed = testbed.run(
problem_size,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta),
raster_order,
detail::MaxSwizzleSize(max_swizzle_size),
detail::Splits(splits)
);
std::vector<int> problem_splits = {1};
if (UsesStreamKScheduler && (decomp_mode == DecompositionMode::Heuristic || decomp_mode == DecompositionMode::SplitK)) {
auto max_splits = (k + TileShapeK - 1) / TileShapeK;
if (max_splits > 2) {
problem_splits.push_back(2);
}
if (max_splits > 3) {
problem_splits.push_back(3);
}
if (!passed) {
return false;
problem_splits.push_back(max_splits);
// Test the case in which we ask for more splits than there are K tiles in the GEMM. In this
// case, split-K will fall back to a splitting factor of `max_splits`.
problem_splits.push_back(max_splits + 1);
}
}
}
}
}
}
}
for (int splits : problem_splits) {
ProblemShapeType problem_size;
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
problem_size = ProblemShapeType{m, n, k, /* l */ 1};
}
else {
problem_size = ProblemShapeType{m, n, k};
}
passed = testbed.run(
problem_size,
cutlass::from_real<ElementScalar>(alpha),
cutlass::from_real<ElementScalar>(beta),
raster_order,
detail::MaxSwizzleSize(max_swizzle_size),
detail::Splits(splits),
decomp_mode
);
if (!passed) {
return false;
}
} // splits
} // decomposition_mode
} // max_swizzle_size
} // raster_order
} // k
} // n
} // m
// if we do support batched GEMM, just run one test on it to save on test time
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
test/unit/gemm/device/gemm_testbed_3x_evt.hpp
+1 -1
View File
@@ -382,7 +382,7 @@ public:
HostAuxLoad(){}
template<typename ProblemShapeType>
HostAuxLoad(ProblemShapeType problem_size, TestBedImpl impl, bool check_relative_equality=false)
: Base(check_relative_equality), impl_(impl){
: Base(check_relative_equality), impl_(impl) {
auto problem_shape_NMKL = cute::append<4>(problem_size, 1);
auto [_M, _N, K, _L] = problem_shape_NMKL;
auto aux_coord = cutlass::make_Coord(_M * _L, _N);
test/unit/gemm/device/sm90_evt_operations.hpp
+7 -7
View File
@@ -267,7 +267,7 @@ template<
using Sm90LinCombAuxLoad =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + (alpha * acc + bias)
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
@@ -295,7 +295,7 @@ template<
using Sm90LinCombEVTDAG =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + (alpha * acc + aux)
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90TopologicalVisitor<
ElementCompute,
cute::tuple<
@@ -349,7 +349,7 @@ using Sm90LinCombDAGEVT =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>,
Sm90ScalarBroadcast<ElementScalar>,
Sm90AccFetch,
Sm90SrcFetch
Sm90SrcFetch<ElementOutput>
>
>,
Sm90ColBroadcast<0, typename EpilogueDescriptor::TileShape, ElementBias>,
@@ -371,7 +371,7 @@ template<
using Sm90LinCombPerColumnBias =
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + (alpha * acc + bias)
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc + bias
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch, // acc
@@ -403,7 +403,7 @@ using Sm90LinCombPerColumnReduce =
Sm90EVT<Sm90RowReduction<RegReduceFn, GmemReduceFn, 0, CtaTileShapeMNK, ElementReduce, ElementCompute, RoundStyle>, // per column reduce
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<multiplies, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch // acc
@@ -428,7 +428,7 @@ using Sm90LinCombPerRowReduce =
Sm90EVT<Sm90ColReduction<RegReduceFn, GmemReduceFn, 0, CtaTileShapeMNK, ElementReduce, ElementCompute, RoundStyle>, // per column reduce
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<multiplies, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch // acc
@@ -452,7 +452,7 @@ using Sm90LinCombScalarReduce =
Sm90EVT<Sm90ScalarReduction<RegReduceFn, GmemReduceFn, ElementReduce, ElementCompute, RoundStyle>, // per column reduce
Sm90EVT<Sm90Compute<homogeneous_multiply_add, ElementOutput, ElementCompute, RoundStyle>, // beta * C + alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // beta
Sm90SrcFetch, // C
Sm90SrcFetch<ElementOutput>, // C
Sm90EVT<Sm90Compute<multiplies, ElementCompute, ElementCompute, RoundStyle>, // alpha * acc
Sm90ScalarBroadcast<ElementScalar>, // alpha
Sm90AccFetch // acc
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative_bias_elementwise.cu
+5 -5
View File
@@ -389,7 +389,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, float>;
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, float, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -434,7 +434,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, cutlass::half_t>;
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, cutlass::half_t, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -480,7 +480,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
// ReLU with uint1b_t aux will compute dReLU/dZ as the aux output, i.e. Aux(i) = (Z(i) >= 0) ? 1 : 0
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLU, cutlass::half_t, float, cutlass::uint1b_t, int8_t>;
LayoutC, cutlass::epilogue::thread::ReLU, cutlass::half_t, float, cutlass::uint1b_t, int8_t, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -525,7 +525,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using FusionOperation = cutlass::epilogue::fusion::LinCombDeEltActDePerRowBias<
LayoutC, cutlass::epilogue::thread::dReLU, cutlass::half_t, float, cutlass::uint1b_t, float>;
LayoutC, cutlass::epilogue::thread::dReLU, cutlass::half_t, float, cutlass::uint1b_t, float, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -570,7 +570,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_epilogue, 25
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative;
using FusionOperation = cutlass::epilogue::fusion::LinCombDeEltAct<
LayoutC, cutlass::epilogue::thread::dGELU, cutlass::half_t, float, cutlass::half_t>;
LayoutC, cutlass::epilogue::thread::dGELU, cutlass::half_t, float, cutlass::half_t, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_pingpong_bias_elementwise.cu
+4 -4
View File
@@ -335,7 +335,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, float>;
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, float, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -380,7 +380,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, cutlass::half_t>;
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, cutlass::half_t, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -425,7 +425,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_persistent_epilogue, 128
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionOperation = cutlass::epilogue::fusion::LinCombPerRowBiasEltActAux<
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, int8_t>;
LayoutC, cutlass::epilogue::thread::ReLu, cutlass::half_t, float, cutlass::half_t, int8_t, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
@@ -470,7 +470,7 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_pingpong_epilogue, 128x1
using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized;
using FusionOperation = cutlass::epilogue::fusion::LinCombDeEltActDePerRowBias<
LayoutC, cutlass::epilogue::thread::dReLU, cutlass::half_t, float, cutlass::uint1b_t, float>;
LayoutC, cutlass::epilogue::thread::dReLU, cutlass::half_t, float, cutlass::uint1b_t, float, void>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
test/unit/gemm/device/sm90_gemm_f16_f16_f16_tensor_op_f32_cooperative_stream_k.cu
+46 -24
View File
@@ -95,7 +95,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x64_1x2x1) {
@@ -136,7 +137,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -178,7 +180,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -218,7 +221,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -258,7 +262,8 @@ TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -298,7 +303,8 @@ TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -341,7 +347,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -381,7 +388,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -421,7 +429,8 @@ TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -461,7 +470,8 @@ TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
@@ -505,7 +515,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -545,7 +556,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -585,7 +597,8 @@ TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -625,7 +638,8 @@ TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 12
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
@@ -669,7 +683,8 @@ TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -709,7 +724,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -749,7 +765,8 @@ TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -789,7 +806,8 @@ TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 25
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16n_f16n_tensor_op_gmma_f32_cooperative_stream_k_epilogue, 256x128x64_2x2x1) {
@@ -827,7 +845,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f16n_tensor_op_gmma_f32_cooperative_stream_k_epi
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16n_f16t_tensor_op_gmma_f32_cooperative_stream_k_epilogue, 256x128x64_2x2x1) {
@@ -865,7 +884,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f16t_tensor_op_gmma_f32_cooperative_stream_k_epi
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k_epilogue, 128x128x64_2x2x1) {
@@ -903,7 +923,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_cooperative_stream_k_epi
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_stream_k_epilogue, 128x128x64_2x2x1) {
@@ -941,7 +962,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_stream_k_epi
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_stream_k_epilogue, 256x128x64_2x2x1_BiasF32_ReLU) {
@@ -985,8 +1007,8 @@ TEST(SM90_Device_Gemm_f16t_f16n_f32t_tensor_op_gmma_f32_cooperative_stream_k_epi
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
bool passed = test::gemm::device::TestAllBiasElementwise<Gemm>();
EXPECT_TRUE(passed);
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_f8_f8_f32_tensor_op_f32_cooperative_stream_k.cu
+20 -10
View File
@@ -99,7 +99,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x128_1x1x1) {
@@ -146,7 +147,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -197,7 +199,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x128_1x2x1) {
@@ -244,7 +247,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -295,7 +299,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x128_1x4x1) {
@@ -342,7 +347,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -393,7 +399,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x128_4x1x1) {
@@ -440,7 +447,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
///////////////////////////////////////////////////////////////////////////////
@@ -491,7 +499,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k, 256x128x128_2x4x1_fp8_fast_accum) {
@@ -538,7 +547,8 @@ TEST(SM90_Device_Gemm_e4m3t_e4m3n_f32n_tensor_op_gmma_f32_cooperative_stream_k,
>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>());
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 0.0));
EXPECT_TRUE(test::gemm::device::TestAllBiasElementwise<Gemm>(1.0, 1.0));
}
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
test/unit/gemm/device/sm90_gemm_stream_k_scheduler.cu
+34 -23
View File
@@ -101,8 +101,34 @@ test_scheduler(
cutlass::KernelHardwareInfo hw_info{0, sm_count};
auto params = Scheduler::to_underlying_arguments(problem_shape_mnkl, tile_shape, cluster_shape, hw_info, {splits}, nullptr);
typename Scheduler::Arguments args{};
// Set up the grid for the problem
dim3 grid = Scheduler::get_grid_shape(problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
auto print_info = [&]() {
std::cout << "Failed with problem size "
<< size<0>(problem_shape_mnkl) << "x"
<< size<1>(problem_shape_mnkl) << "x"
<< size<2>(problem_shape_mnkl) << "x"
<< size<3>(problem_shape_mnkl)
<< " and grid size " << grid.x << "x"
<< grid.y << "x" << grid.z
<< " splits=" << params.splits_
<< " k_iter=" << params.divmod_tiles_per_output_tile_.divisor
<< " big_units_=" << params.big_units_
<< " big_groups_=" << params.big_groups_
<< " sk_tiles=" << params.sk_tiles_
<< " sk_units=" << params.sk_units_
<< " k_tiles_per_sk_unit=" << params.k_tiles_per_sk_unit_
<< " units_per_problem=" << params.units_per_problem_
<< " groups=" << params.divmod_sk_groups_.divisor << std::endl;
};
// If we expect the schedule to be data-parallel only, ensure that no stream-K tiles are launched.
if (expect_data_parallel && params.sk_tiles_ != 0) {
print_info();
std::cout << "Expected stream-K to select a data-parallel decomposition." << std::endl;
return false;
}
@@ -114,15 +140,11 @@ test_scheduler(
// Initialize counters to zero
cudaError_t err = cudaMemset((void*)visit_counters.get(), 0, sizeof(int) * total_counters);
if (err != cudaSuccess) {
std::cerr << __FILE__ << ":" << __LINE__ << " cudaMemset failed with error: " << cudaGetErrorString(err) << std::endl;
print_info();
std::cout << __FILE__ << ":" << __LINE__ << " cudaMemset failed with error: " << cudaGetErrorString(err) << std::endl;
return false;
}
typename Scheduler::Arguments args{};
// Set up the grid for the problem
dim3 grid = Scheduler::get_grid_shape(problem_shape_mnkl, tile_shape, cluster_shape, hw_info, args);
// Set up cluster and cluster launch. This is needed even for this simple kernel because
// the SM90 scheduler needs to be able to query the CTA id within a cluster, which requires
// explicitly launching with clusters.
@@ -161,7 +183,8 @@ test_scheduler(
err = cudaLaunchKernelExC(&launch_config, kernel, kernel_params);
if (err != cudaSuccess) {
std::cerr << __FILE__ << ":" << __LINE__
print_info();
std::cout << __FILE__ << ":" << __LINE__
<< " cudaLaunchKernelExC failed with error: "
<< cudaGetErrorString(err) << std::endl;
return false;
@@ -169,7 +192,8 @@ test_scheduler(
err = cudaDeviceSynchronize();
if (err != cudaSuccess) {
std::cerr << __FILE__ << ":" << __LINE__
print_info();
std::cout << __FILE__ << ":" << __LINE__
<< " scheduler kernel failed with error: "
<< cudaGetErrorString(err) << std::endl;
return false;
@@ -181,20 +205,7 @@ test_scheduler(
for (size_t i = 0; i < host_visit_counts.size(); ++i) {
if (host_visit_counts[i] != 1) {
std::cout << "Failed with problem size "
<< size<0>(problem_shape_mnkl) << "x"
<< size<1>(problem_shape_mnkl) << "x"
<< size<2>(problem_shape_mnkl) << "x"
<< size<3>(problem_shape_mnkl)
<< " and grid size " << grid.x << "x"
<< grid.y << "x" << grid.z
<< " splits=" << params.splits_
<< " k_iter=" << params.divmod_tiles_per_output_tile_.divisor
<< " big_units=" << params.big_units_
<< " sk_tiles=" << params.sk_tiles_
<< " sk_units=" << params.sk_units_
<< " k_tiles_per_sk_unit=" << params.k_tiles_per_sk_unit_
<< " units_per_problem=" << params.units_per_problem_ << std::endl;
print_info();
std::cout << "Error at idx: " << i << ". Got count " << host_visit_counts[i] << std::endl;
return false;
}
@@ -301,7 +312,7 @@ TEST(SM90_Device_Gemm_stream_k_scheduler, 256x128x64_2x1x1) {
// Test various data-parallel cases
EXPECT_TRUE(test_data_parallel(/*blocks_m=*/ 4, /*blocks_n=*/ 4, tile_shape, cluster_shape, /*sm_count=*/ 16));
EXPECT_TRUE(test_data_parallel(/*blocks_m=*/16, /*blocks_n=*/ 4, tile_shape, cluster_shape, /*sm_count=*/ 64));
EXPECT_TRUE(test_data_parallel(/*blocks_m=*/ 4, /*blocks_n=*/27, tile_shape, cluster_shape, /*sm_count=*/108));
EXPECT_TRUE(test_data_parallel(/*blocks_m=*/ 8, /*blocks_n=*/27, tile_shape, cluster_shape, /*sm_count=*/108));
// Test various stream-K cases
EXPECT_TRUE(test_stream_k(tile_shape, cluster_shape, /*sm_count=*/ 16));
test/unit/nvrtc/thread/nvrtc_contraction.cu
+2 -1
View File
@@ -43,6 +43,7 @@
static_assert(0, "CUDA include path is not defined");
#endif
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
TEST(SM90_nvrtc_kernel, Contraction) {
static const char* nvrtc_opts[] = {
"-w",
@@ -62,5 +63,5 @@ TEST(SM90_nvrtc_kernel, Contraction) {
{ nvrtc_opts, nvrtc_opts + 5 }
));
}
#endif
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/pipeline/pipeline_tma_async.cu
+6 -6
View File
@@ -60,10 +60,10 @@ using namespace cute;
//////////////////// KERNEL /////////////////////////
template <uint32_t Stages, typename ClusterShape>
template <uint32_t Stages>
struct SharedStorage
{
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage storage;
typename cutlass::PipelineTmaAsync<Stages>::SharedStorage storage;
};
// Goal of this kernel is to complete deadlock-free
@@ -73,10 +73,10 @@ void pipeline_device(uint32_t const NumIterations)
{
extern __shared__ char shared_memory[];
using MainloopPipeline = cutlass::PipelineTmaAsync<NumStages, ClusterShape>;
using MainloopPipeline = cutlass::PipelineTmaAsync<NumStages>;
using PipelineState = cutlass::PipelineState<NumStages>;
using SharedStorage = SharedStorage<NumStages, ClusterShape>;
using SharedStorage = SharedStorage<NumStages>;
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
[[maybe_unused]] auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
@@ -98,7 +98,7 @@ void pipeline_device(uint32_t const NumIterations)
params.is_leader = warp_group_thread_idx == 0;
params.num_consumers = 128;
MainloopPipeline pipeline(shared_storage.storage, params);
MainloopPipeline pipeline(shared_storage.storage, params, cluster_shape);
__syncthreads();
@@ -223,7 +223,7 @@ struct PipelineTest {
}
for (int iter = 0; iter < iterations; ++iter) {
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape)>));
int smem_size = int(sizeof(SharedStorage<Stages>));
result = cudaFuncSetAttribute(
pipeline_device<decltype(cluster_shape), Stages>,
test/unit/pipeline/pipeline_tma_async_warp_specialized.cu
+7 -7
View File
@@ -62,10 +62,10 @@ using namespace cutlass;
//////////////////// KERNEL /////////////////////////
template <uint32_t Stages, typename ClusterShape>
template <uint32_t Stages>
struct SharedStorage
{
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage storage ;
typename cutlass::PipelineTmaAsync<Stages>::SharedStorage storage ;
};
struct KernelParams
@@ -81,10 +81,10 @@ __global__ static
void pipeline_device(KernelParams const kernel_params)
{
extern __shared__ char shared_memory[];
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages>;
using PipelineState = typename cutlass::PipelineState<Stages>;
using SharedStorage = SharedStorage<Stages, ClusterShape>;
using SharedStorage = SharedStorage<Stages>;
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
[[maybe_unused]] auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
@@ -112,7 +112,7 @@ void pipeline_device(KernelParams const kernel_params)
params.is_leader = warp_group_thread_idx == 0;
params.num_consumers = 128;
MainloopPipeline pipeline(shared_storage.storage, params);
MainloopPipeline pipeline(shared_storage.storage, params, cluster_shape);
__syncthreads();
@@ -292,9 +292,9 @@ struct PipelineTest {
for (int iter = 0; iter < iterations; ++iter) {
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, decltype(cluster_shape)>;
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages>;
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape)>));
int smem_size = int(sizeof(SharedStorage<Stages>));
result = cudaFuncSetAttribute(
pipeline_device<decltype(cluster_shape), Stages>,
test/unit/pipeline/pipeline_tma_async_warp_specialized_persistent.cu
+7 -7
View File
@@ -62,16 +62,16 @@ using namespace cutlass;
//////////////////// KERNEL /////////////////////////
template <uint32_t Stages, typename ClusterShape, typename PingPongBarrier>
template <uint32_t Stages, typename PingPongBarrier>
struct SharedStorage
{
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage pipeline_storage;
typename cutlass::PipelineTmaAsync<Stages>::SharedStorage pipeline_storage;
typename PingPongBarrier::SharedStorage pingpong_storage;
};
template <typename ClusterShape, uint32_t Stages>
struct CollectiveSimulation {
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages>;
using PipelineState = typename cutlass::PipelineState<Stages>;
CUTLASS_DEVICE
@@ -198,7 +198,7 @@ __global__ static
void pipeline_device(KernelParams params)
{
extern __shared__ char shared_memory[];
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages>;
using PipelineState = typename cutlass::PipelineState<Stages>;
/* One for Mainloop and one for Epilogue */
@@ -206,7 +206,7 @@ void pipeline_device(KernelParams params)
constexpr int MathWarpGroupCountPersistent = 2;
using PingPongBarrier = typename cutlass::OrderedSequenceBarrier<StagesPerMathWarpGroup, MathWarpGroupCountPersistent>;
using SharedStorage = SharedStorage<Stages, ClusterShape, PingPongBarrier>;
using SharedStorage = SharedStorage<Stages, PingPongBarrier>;
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
[[maybe_unused]] auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
@@ -232,7 +232,7 @@ void pipeline_device(KernelParams params)
pipeline_params.is_leader = warp_group_thread_idx == 0;
pipeline_params.num_consumers = NumThreadsPerWarpGroup;
MainloopPipeline pipeline(shared_storage.pipeline_storage, pipeline_params);
MainloopPipeline pipeline(shared_storage.pipeline_storage, pipeline_params, cluster_shape);
PipelineState tile_start_state_pipe;
int tiles_per_cluster = params.tiles_per_cluster;
@@ -343,7 +343,7 @@ struct PipelineTest {
for (int iter = 0; iter < iterations; ++iter) {
constexpr int StagesPerMathWarpGroup = 2;
constexpr int MathWarpGroupCountPersistent = 2;
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape),
int smem_size = int(sizeof(SharedStorage<Stages,
typename cutlass::OrderedSequenceBarrier<StagesPerMathWarpGroup, MathWarpGroupCountPersistent>>));
result = cudaFuncSetAttribute(