Updates for 3.1 (#932)
This commit is contained in:
ANIKET SHIVAM
@@ -182,9 +182,9 @@ class GemmOperation:
|
||||
ar = self.arch,
|
||||
op = opcode_class_name,
|
||||
ex = self.extended_name_3x(),
|
||||
tbm = self.tile_description.threadblock_shape[0],
|
||||
tbn = self.tile_description.threadblock_shape[1],
|
||||
tbk = self.tile_description.threadblock_shape[2],
|
||||
tbm = self.tile_description.tile_shape[0],
|
||||
tbn = self.tile_description.tile_shape[1],
|
||||
tbk = self.tile_description.tile_shape[2],
|
||||
cm = self.tile_description.cluster_shape[0],
|
||||
cn = self.tile_description.cluster_shape[1],
|
||||
ck = self.tile_description.cluster_shape[2],
|
||||
@@ -640,7 +640,7 @@ class EmitGemmUniversal3xInstance:
|
||||
using ${operation_name}_epilogue =
|
||||
typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
${arch}, ${opcode_class},
|
||||
cute::Shape<cute::_${threadblock_shape_m}, cute::_${threadblock_shape_n}, cute::_${threadblock_shape_k}>,
|
||||
cute::Shape<cute::_${tile_shape_m}, cute::_${tile_shape_n}, cute::_${tile_shape_k}>,
|
||||
cute::Shape<cute::_${cluster_m},cute::_${cluster_n},cute::_${cluster_k}>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
${element_accumulator}, ${element_epilogue},
|
||||
@@ -655,7 +655,7 @@ using ${operation_name}_mainloop =
|
||||
${element_a}, ${layout_a}, ${align_a},
|
||||
${element_b}, ${layout_b}, ${align_b},
|
||||
${element_accumulator},
|
||||
cute::Shape<cute::_${threadblock_shape_m}, cute::_${threadblock_shape_n}, cute::_${threadblock_shape_k}>,
|
||||
cute::Shape<cute::_${tile_shape_m}, cute::_${tile_shape_n}, cute::_${tile_shape_k}>,
|
||||
cute::Shape<cute::_${cluster_m},cute::_${cluster_n},cute::_${cluster_k}>,
|
||||
cutlass::gemm::collective::StageCountAutoCarveout<
|
||||
sizeof(typename ${operation_name}_epilogue::SharedStorage)>,
|
||||
@@ -686,14 +686,14 @@ ${compile_guard_end}
|
||||
#
|
||||
def emit(self, operation):
|
||||
|
||||
threadblock_shape = operation.tile_description.threadblock_shape
|
||||
tile_shape = operation.tile_description.tile_shape
|
||||
warp_count = operation.tile_description.warp_count
|
||||
# stage count set to zero indicates builder automatic stage selection
|
||||
if operation.tile_description.stages > 0:
|
||||
stage_count_string = f"cutlass::gemm::collective::StageCount<{str(operation.tile_description.stages)}>"
|
||||
else:
|
||||
stage_count_string = "cutlass::gemm::collective::StageCountAuto"
|
||||
warp_shape = [threadblock_shape[idx] // warp_count[idx] for idx in range(3)]
|
||||
warp_shape = [tile_shape[idx] // warp_count[idx] for idx in range(3)]
|
||||
|
||||
instance_layout_A, instance_layout_B, instance_layout_C , instance_layout_D = \
|
||||
(operation.A.layout, operation.B.layout, operation.C.layout, operation.D.layout)
|
||||
@@ -727,9 +727,9 @@ ${compile_guard_end}
|
||||
'element_accumulator': DataTypeTag[operation.accumulator_type()],
|
||||
'opcode_class': OpcodeClassTag[operation.tile_description.math_instruction.opcode_class],
|
||||
'arch': "cutlass::arch::Sm%d" % operation.arch,
|
||||
'threadblock_shape_m': str(operation.tile_description.threadblock_shape[0]),
|
||||
'threadblock_shape_n': str(operation.tile_description.threadblock_shape[1]),
|
||||
'threadblock_shape_k': str(operation.tile_description.threadblock_shape[2]),
|
||||
'tile_shape_m': str(operation.tile_description.tile_shape[0]),
|
||||
'tile_shape_n': str(operation.tile_description.tile_shape[1]),
|
||||
'tile_shape_k': str(operation.tile_description.tile_shape[2]),
|
||||
'cluster_m': str(operation.tile_description.cluster_shape[0]),
|
||||
'cluster_n': str(operation.tile_description.cluster_shape[1]),
|
||||
'cluster_k': str(operation.tile_description.cluster_shape[2]),
|
||||
|
||||
@@ -91,22 +91,21 @@ def CreateGemmOperator(manifest, layouts, tile_descriptions, data_type, \
|
||||
|
||||
# Generates 3.0 API based GemmUniversal API kernels. Alignment constraints are folded in with layouts
|
||||
def CreateGemmUniversal3xOperator(
|
||||
manifest, layouts, tile_descriptions, data_type,
|
||||
manifest, layouts, tile_descriptions, data_types,
|
||||
schedules = [[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.ScheduleAuto]],
|
||||
complex_transforms=None,
|
||||
epilogue_functor=EpilogueFunctor.LinearCombination,
|
||||
swizzling_functor=SwizzlingFunctor.Identity1):
|
||||
|
||||
if type(data_types) is dict:
|
||||
data_types = [data_types]
|
||||
|
||||
for s in schedules:
|
||||
assert(len(s) == 2)
|
||||
|
||||
if complex_transforms is None:
|
||||
complex_transforms = [(ComplexTransform.none, ComplexTransform.none), ]
|
||||
|
||||
element_a = data_type["a_type"]
|
||||
element_b = data_type["b_type"]
|
||||
element_c = data_type["c_type"]
|
||||
element_d = data_type["d_type"]
|
||||
element_acc = data_type["acc_type"]
|
||||
element_epilogue = data_type.get("epi_type", element_acc)
|
||||
|
||||
operations = []
|
||||
|
||||
# by default, only generate the largest tile and largest alignment
|
||||
@@ -115,23 +114,25 @@ def CreateGemmUniversal3xOperator(
|
||||
|
||||
for layout in layouts:
|
||||
for tile_description in tile_descriptions:
|
||||
for complex_transform in complex_transforms:
|
||||
for kernel_schedule, epilogue_schedule in schedules:
|
||||
A = TensorDescription(
|
||||
element_a, layout[0][0], layout[0][1], complex_transform[0])
|
||||
B = TensorDescription(
|
||||
element_b, layout[1][0], layout[1][1], complex_transform[1])
|
||||
for data_type in data_types:
|
||||
for complex_transform in complex_transforms:
|
||||
for kernel_schedule, epilogue_schedule in schedules:
|
||||
A = TensorDescription(
|
||||
data_type["a_type"], layout[0][0], layout[0][1], complex_transform[0])
|
||||
B = TensorDescription(
|
||||
data_type["b_type"], layout[1][0], layout[1][1], complex_transform[1])
|
||||
|
||||
C = TensorDescription(element_c, layout[2][0], layout[2][1])
|
||||
D = TensorDescription(element_d, layout[2][0], layout[2][1])
|
||||
C = TensorDescription(data_type["c_type"], layout[2][0], layout[2][1])
|
||||
D = TensorDescription(data_type["d_type"], layout[2][0], layout[2][1])
|
||||
|
||||
operation = GemmOperation(
|
||||
GemmKind.Universal3x, tile_description.minimum_compute_capability,
|
||||
tile_description, A, B, C, element_epilogue, epilogue_functor, swizzling_functor, D,
|
||||
kernel_schedule, epilogue_schedule)
|
||||
element_compute = data_type.get("epi_type", data_type["acc_type"])
|
||||
operation = GemmOperation(
|
||||
GemmKind.Universal3x, tile_description.minimum_compute_capability,
|
||||
tile_description, A, B, C, element_compute, epilogue_functor, swizzling_functor, D,
|
||||
kernel_schedule, epilogue_schedule)
|
||||
|
||||
manifest.append(operation)
|
||||
operations.append(operation)
|
||||
manifest.append(operation)
|
||||
operations.append(operation)
|
||||
|
||||
return operations
|
||||
|
||||
@@ -4118,21 +4119,19 @@ def GenerateSM90_TensorOp_16b_WGMMA_gemm(manifest, cuda_version):
|
||||
layout[2][1] = 8
|
||||
|
||||
if CudaToolkitVersionSatisfies(cuda_version, 12, 1):
|
||||
kernel_schedules = [
|
||||
KernelScheduleType.ScheduleAuto,
|
||||
KernelScheduleType.TmaWarpSpecializedCooperative,
|
||||
KernelScheduleType.TmaWarpSpecializedPingpong,
|
||||
KernelScheduleType.TmaWarpSpecialized
|
||||
schedules = [
|
||||
[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.ScheduleAuto],
|
||||
[KernelScheduleType.TmaWarpSpecializedCooperative, EpilogueScheduleType.NoSmemWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecializedPingpong, EpilogueScheduleType.NoSmemWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecialized, EpilogueScheduleType.NoSmemWarpSpecialized]
|
||||
]
|
||||
else:
|
||||
kernel_schedules = [
|
||||
KernelScheduleType.ScheduleAuto,
|
||||
KernelScheduleType.TmaWarpSpecialized
|
||||
schedules = [
|
||||
[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.ScheduleAuto],
|
||||
[KernelScheduleType.TmaWarpSpecialized, EpilogueScheduleType.NoSmemWarpSpecialized]
|
||||
# TmaWarpSpecializedCooperative and TmaWarpSpecializedPingpong require CUDA version >= 12.1 for optimal performance.
|
||||
]
|
||||
|
||||
schedules = [[s, EpilogueScheduleType.ScheduleAuto] for s in kernel_schedules]
|
||||
|
||||
CreateGemmUniversal3xOperator(manifest, layouts, tile_descriptions, data_type, schedules)
|
||||
|
||||
# persistent kernels with TMA epilogues
|
||||
@@ -4140,6 +4139,11 @@ def GenerateSM90_TensorOp_16b_WGMMA_gemm(manifest, cuda_version):
|
||||
CreateGemmUniversal3xOperator(manifest, layouts, tile_descriptions, data_type,
|
||||
[[KernelScheduleType.TmaWarpSpecializedPingpong, EpilogueScheduleType.TmaWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecializedCooperative, EpilogueScheduleType.TmaWarpSpecializedCooperative]])
|
||||
# Emit instance without C allocation+load
|
||||
data_type["c_type"] = DataType.void
|
||||
CreateGemmUniversal3xOperator(manifest, layouts, tile_descriptions, data_type,
|
||||
[[KernelScheduleType.TmaWarpSpecializedPingpong, EpilogueScheduleType.TmaWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecializedCooperative, EpilogueScheduleType.TmaWarpSpecializedCooperative]])
|
||||
|
||||
# for mixed precision kernels, also generate kernels that write output matrix in the A/B format
|
||||
# Avoid emitting two kernels if the accumulator type does not differ from the input type (e.g. F16 accumulation)
|
||||
@@ -4166,6 +4170,11 @@ def GenerateSM90_TensorOp_16b_WGMMA_gemm(manifest, cuda_version):
|
||||
CreateGemmUniversal3xOperator(manifest, layouts, tile_descriptions, data_type_mixed,
|
||||
[[KernelScheduleType.TmaWarpSpecializedPingpong, EpilogueScheduleType.TmaWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecializedCooperative, EpilogueScheduleType.TmaWarpSpecializedCooperative]])
|
||||
# Emit instance without C allocation+load
|
||||
data_type_mixed["c_type"] = DataType.void
|
||||
CreateGemmUniversal3xOperator(manifest, layouts, tile_descriptions, data_type_mixed,
|
||||
[[KernelScheduleType.TmaWarpSpecializedPingpong, EpilogueScheduleType.TmaWarpSpecialized],
|
||||
[KernelScheduleType.TmaWarpSpecializedCooperative, EpilogueScheduleType.TmaWarpSpecializedCooperative]])
|
||||
|
||||
#
|
||||
def GenerateSM90_TensorOp_tf32_WGMMA_gemm(manifest, cuda_version):
|
||||
@@ -4212,19 +4221,32 @@ def GenerateSM90_TensorOp_tf32_WGMMA_gemm(manifest, cuda_version):
|
||||
"acc_type" : math_inst.element_accumulator,
|
||||
"epi_type" : math_inst.element_accumulator
|
||||
}
|
||||
|
||||
schedules = [
|
||||
[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.ScheduleAuto],
|
||||
[KernelScheduleType.TmaWarpSpecialized, EpilogueScheduleType.NoSmemWarpSpecialized]
|
||||
]
|
||||
|
||||
# TMA kernels with TT layout use EpilogueTransposed (NoSmemWarpSpecialized with swapped strides),
|
||||
# because they use NN kernels underneath and transposing its epilogue will get the correct output
|
||||
schedules_transposed_epilogue = [
|
||||
[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.EpilogueTransposed],
|
||||
[KernelScheduleType.TmaWarpSpecialized, EpilogueScheduleType.EpilogueTransposed]
|
||||
]
|
||||
|
||||
# TMA kernels with TN or NN layout
|
||||
layouts_tf32_tn_nn = [layouts_tf32[0], layouts_tf32[2]]
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tn_nn, tile_descriptions, data_type_tf32)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tn_nn, tile_descriptions, data_type_tf32, schedules)
|
||||
|
||||
# TMA kernels with NT layout, only support 64x128x32 tile for now.
|
||||
layouts_tf32_nt = [layouts_tf32[3]]
|
||||
tile_64x128x32_descriptions = [tile_descriptions[0], tile_descriptions[1], tile_descriptions[2]]
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_64x128x32_descriptions, data_type_tf32)
|
||||
tile_128x128x32_descriptions = [tile_descriptions[3], tile_descriptions[4], tile_descriptions[5]]
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_64x128x32_descriptions, data_type_tf32, schedules)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_128x128x32_descriptions, data_type_tf32, [schedules[1]])
|
||||
|
||||
# TMA kernels with TT layout use EpilogueTransposed, because swapping NN kernel and transposed its epilogue will get the kernel
|
||||
layouts_tf32_tt = [layouts_tf32[1]]
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tt, tile_descriptions, data_type_tf32,
|
||||
[[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.EpilogueTransposed]])
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tt, tile_descriptions, data_type_tf32, schedules_transposed_epilogue)
|
||||
|
||||
# F32 kernel share same settings with tf32 I/O kernels excluding data type
|
||||
data_type_f32 = {
|
||||
@@ -4236,10 +4258,10 @@ def GenerateSM90_TensorOp_tf32_WGMMA_gemm(manifest, cuda_version):
|
||||
"epi_type" : DataType.f32
|
||||
}
|
||||
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tn_nn, tile_descriptions, data_type_f32)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_64x128x32_descriptions, data_type_f32)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tt, tile_descriptions, data_type_f32,
|
||||
[[KernelScheduleType.ScheduleAuto, EpilogueScheduleType.EpilogueTransposed]])
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tn_nn, tile_descriptions, data_type_f32, schedules)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_64x128x32_descriptions, data_type_f32, schedules)
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_nt, tile_128x128x32_descriptions, data_type_f32, [schedules[1]])
|
||||
CreateGemmUniversal3xOperator(manifest, layouts_tf32_tt, tile_descriptions, data_type_f32, schedules_transposed_epilogue)
|
||||
|
||||
#
|
||||
def GenerateSM90_TensorOp_int8_WGMMA_gemm(manifest, cuda_version):
|
||||
@@ -4910,8 +4932,8 @@ def GenerateSM90_TensorOp_1684_symm_complex_gaussian(manifest, cuda_version):
|
||||
#
|
||||
def GenerateSM90(manifest, cuda_version):
|
||||
GenerateSM90_TensorOp_16b_WGMMA_gemm(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_int8_WGMMA_gemm(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_tf32_WGMMA_gemm(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_int8_WGMMA_gemm(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_1684(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_1684_complex(manifest, cuda_version)
|
||||
GenerateSM90_TensorOp_1684_complex_gaussian(manifest, cuda_version)
|
||||
|
||||
@@ -40,6 +40,7 @@ GeneratorTargetNames = {
|
||||
|
||||
#
|
||||
class DataType(enum.Enum):
|
||||
void = enum_auto() # primarily used to disable C tensor for epilogues
|
||||
b1 = enum_auto()
|
||||
u4 = enum_auto()
|
||||
u8 = enum_auto()
|
||||
@@ -89,6 +90,7 @@ ShortDataTypeNames = {
|
||||
|
||||
#
|
||||
DataTypeNames = {
|
||||
DataType.void: "void",
|
||||
DataType.b1: "b1",
|
||||
DataType.u4: "u4",
|
||||
DataType.u8: "u8",
|
||||
@@ -121,10 +123,11 @@ DataTypeNames = {
|
||||
DataType.cs8: "cs8",
|
||||
DataType.cs16: "cs16",
|
||||
DataType.cs32: "cs32",
|
||||
DataType.cs64: "cs64",
|
||||
DataType.cs64: "cs64",
|
||||
}
|
||||
|
||||
DataTypeTag = {
|
||||
DataType.void: "void",
|
||||
DataType.b1: "cutlass::uint1b_t",
|
||||
DataType.u4: "cutlass::uint4b_t",
|
||||
DataType.u8: "uint8_t",
|
||||
@@ -161,6 +164,7 @@ DataTypeTag = {
|
||||
}
|
||||
|
||||
DataTypeSize = {
|
||||
DataType.void: 0,
|
||||
DataType.b1: 1,
|
||||
DataType.u4: 4,
|
||||
DataType.u8: 8,
|
||||
@@ -765,6 +769,7 @@ class TileDescription:
|
||||
|
||||
def __init__(self, threadblock_shape, stages, warp_count, math_instruction, min_compute, max_compute, cluster_shape = [1,1,1]):
|
||||
self.threadblock_shape = threadblock_shape
|
||||
self.tile_shape = threadblock_shape
|
||||
self.stages = stages
|
||||
self.warp_count = warp_count
|
||||
self.math_instruction = math_instruction
|
||||
|
||||
@@ -240,7 +240,9 @@ class Manifest:
|
||||
self.kernel_filter_list = []
|
||||
else:
|
||||
self.kernel_filter_list = self.get_kernel_filters(args.kernel_filter_file)
|
||||
|
||||
_LOGGER.info("Using {filter_count} kernel filters from {filter_file}".format(
|
||||
filter_count = len(self.kernel_filter_list),
|
||||
filter_file = args.kernel_filter_file))
|
||||
|
||||
self.operation_count = 0
|
||||
self.operations_by_name = {}
|
||||
@@ -311,19 +313,29 @@ class Manifest:
|
||||
# compare against the include list
|
||||
for name_substr in self.kernel_names:
|
||||
if self._filter_string_matches(name_substr, name):
|
||||
_LOGGER.debug("Kernel {kernel} included due to filter string '{filt}'.".format(
|
||||
kernel = operation.procedural_name(),
|
||||
filt = name_substr))
|
||||
enabled = True
|
||||
break
|
||||
|
||||
# compare against the exclude list
|
||||
for name_substr in self.ignore_kernel_names:
|
||||
if self._filter_string_matches(name_substr, name):
|
||||
_LOGGER.debug("Kernel {kernel} ignored due to filter string '{filt}'.".format(
|
||||
kernel = operation.procedural_name(),
|
||||
filt = name_substr))
|
||||
enabled = False
|
||||
break
|
||||
|
||||
if len(self.kernel_filter_list) > 0:
|
||||
enabled = False
|
||||
if self.filter_out_kernels(operation.procedural_name(), self.kernel_filter_list):
|
||||
enabled = True
|
||||
_LOGGER.debug("Kernel {kernel} matched via kernel filter file.".format(kernel = operation.procedural_name()))
|
||||
enabled = True
|
||||
else:
|
||||
_LOGGER.debug("Kernel {kernel} culled due to no match in kernel filter file.".format(kernel = operation.procedural_name()))
|
||||
enabled = False
|
||||
|
||||
|
||||
# todo: filter based on compute data type
|
||||
return enabled
|
||||
@@ -389,6 +401,8 @@ class Manifest:
|
||||
for operation_kind, configurations in self.operations.items():
|
||||
with operation_emitters[target](generated_path, operation_kind, self.args) as operation_kind_emitter:
|
||||
for configuration_name, operations in configurations.items():
|
||||
_LOGGER.info("Emitting {config} with {num_ops} operations.".format(
|
||||
config = configuration_name, num_ops = len(operations)))
|
||||
operation_kind_emitter.emit(configuration_name, operations)
|
||||
|
||||
source_files += operation_kind_emitter.source_files
|
||||
|
||||
@@ -64,6 +64,10 @@ namespace library {
|
||||
|
||||
template <typename T> struct NumericTypeMap;
|
||||
|
||||
template <> struct NumericTypeMap<void> {
|
||||
static NumericTypeID const kId = NumericTypeID::kVoid;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::uint1b_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kB1;
|
||||
};
|
||||
|
||||
@@ -107,15 +107,17 @@ set(CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_SYMM --operation=Symm --provid
|
||||
cutlass_add_executable_tests(
|
||||
test_profiler cutlass_profiler
|
||||
DEPENDEES test_all
|
||||
TEST_COMMAND_OPTIONS
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_GEMM
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV2D
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_CONV3D
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_SPGEMM
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_RANK_K
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_RANK_2K
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_TRMM
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_SYMM
|
||||
TEST_COMMAND_OPTIONS
|
||||
GEMM
|
||||
CONV2D
|
||||
CONV3D
|
||||
SPGEMM
|
||||
RANK_K
|
||||
RANK_2K
|
||||
TRMM
|
||||
SYMM
|
||||
TEST_COMMAND_OPTIONS_PREFIX
|
||||
CUTLASS_PROFILER_TEST_COMMAND_OPTIONS_
|
||||
DISABLE_EXECUTABLE_INSTALL_RULE
|
||||
)
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ int CutlassProfiler::operator()() {
|
||||
options_.execution_mode == ExecutionMode::kTrace) {
|
||||
|
||||
// Profiles all operations
|
||||
profile_();
|
||||
return profile_();
|
||||
}
|
||||
else if (options_.execution_mode == ExecutionMode::kEnumerate) {
|
||||
// Enumerates all operations
|
||||
@@ -157,7 +157,7 @@ int CutlassProfiler::profile_() {
|
||||
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -462,6 +462,13 @@ size_t DeviceAllocation::bytes() const {
|
||||
|
||||
/// Copies from an equivalent-sized tensor in device memory
|
||||
void DeviceAllocation::copy_from_device(void const *ptr) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping copy of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaMemcpy(data(), ptr, bytes(), cudaMemcpyDeviceToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("Failed device-to-device copy");
|
||||
@@ -470,22 +477,43 @@ void DeviceAllocation::copy_from_device(void const *ptr) {
|
||||
|
||||
/// Copies from an equivalent-sized tensor in device memory
|
||||
void DeviceAllocation::copy_from_host(void const *ptr) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping copy of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaMemcpy(data(), ptr, bytes(), cudaMemcpyHostToDevice);
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("Failed device-to-device copy");
|
||||
throw std::runtime_error("Failed host-to-device copy");
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies from an equivalent-sized tensor in device memory
|
||||
void DeviceAllocation::copy_to_host(void *ptr) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping copy of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
cudaError_t result = cudaMemcpy(ptr, data(), bytes(), cudaMemcpyDeviceToHost);
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("Failed device-to-device copy");
|
||||
throw std::runtime_error("Failed device-to-host copy");
|
||||
}
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
if (!good()) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping initialization of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
@@ -690,7 +718,14 @@ void DeviceAllocation::initialize_random_device(int seed, Distribution dist) {
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
if (!good()) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping initialization of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
@@ -699,7 +734,7 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
switch (type_) {
|
||||
case library::NumericTypeID::kFE4M3:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::float_e4m3_t>(
|
||||
reinterpret_cast<cutlass::float_e4m3_t *>(pointer_),
|
||||
reinterpret_cast<cutlass::float_e4m3_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
@@ -707,7 +742,7 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
break;
|
||||
case library::NumericTypeID::kFE5M2:
|
||||
cutlass::reference::host::BlockFillRandom<cutlass::float_e5m2_t>(
|
||||
reinterpret_cast<cutlass::float_e5m2_t *>(pointer_),
|
||||
reinterpret_cast<cutlass::float_e5m2_t *>(host_data.data()),
|
||||
capacity_,
|
||||
seed,
|
||||
dist
|
||||
@@ -904,7 +939,14 @@ void DeviceAllocation::initialize_random_host(int seed, Distribution dist) {
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_sparsemeta_device(int seed, int MetaSizeInBits) {
|
||||
if (!good()) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping initialization of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
@@ -934,7 +976,14 @@ void DeviceAllocation::initialize_random_sparsemeta_device(int seed, int MetaSiz
|
||||
}
|
||||
|
||||
void DeviceAllocation::initialize_random_sparsemeta_host(int seed, int MetaSizeInBits) {
|
||||
if (!good()) {
|
||||
if (!bytes()) {
|
||||
#ifndef NDEBUG
|
||||
std::cout << "Skipping initialization of size 0 allocation\n";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (!data()) {
|
||||
throw std::runtime_error("Attempting to initialize invalid allocation.");
|
||||
}
|
||||
|
||||
|
||||
@@ -68,6 +68,7 @@ GemmOperationProfiler::GemmOperationProfiler(Options const &options):
|
||||
{ArgumentTypeID::kTensor, {"A"}, "Tensor storing the A operand"},
|
||||
{ArgumentTypeID::kTensor, {"B"}, "Tensor storing the B operand"},
|
||||
{ArgumentTypeID::kTensor, {"C"}, "Tensor storing the C operand"},
|
||||
{ArgumentTypeID::kTensor, {"D"}, "Tensor storing the D output"},
|
||||
{ArgumentTypeID::kScalar, {"alpha", "epilogue::alpha"}, "Epilogue scalar alpha"},
|
||||
{ArgumentTypeID::kScalar, {"beta", "epilogue::beta"}, "Epilogue scalar beta"},
|
||||
{ArgumentTypeID::kEnumerated, {"split_k_mode", "split-k-mode"}, "Variant of split K mode(serial, parallel)"},
|
||||
@@ -206,6 +207,10 @@ Status GemmOperationProfiler::GemmProblem::parse(
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!tensor_description_satisfies(operation_desc.D, "D", problem_space, problem)) {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
|
||||
if (!arg_as_scalar(
|
||||
this->alpha,
|
||||
operation_desc.element_epilogue,
|
||||
@@ -307,6 +312,9 @@ void GemmOperationProfiler::GemmProblem::initialize_result(
|
||||
set_argument(result, "C", problem_space,
|
||||
std::string(library::to_string(operation_desc.C.element)) + ":" + library::to_string(operation_desc.C.layout));
|
||||
|
||||
set_argument(result, "D", problem_space,
|
||||
std::string(library::to_string(operation_desc.D.element)) + ":" + library::to_string(operation_desc.D.layout));
|
||||
|
||||
set_argument(result, "m", problem_space, m);
|
||||
set_argument(result, "n", problem_space, n);
|
||||
set_argument(result, "k", problem_space, k);
|
||||
@@ -537,8 +545,6 @@ Status GemmOperationProfiler::initialize_workspace(
|
||||
problem_.batch_count * gemm_workspace_.problem_count
|
||||
);
|
||||
|
||||
gemm_workspace_.Reference->copy_from_device(gemm_workspace_.C->data());
|
||||
|
||||
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
|
||||
gemm_workspace_.arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
|
||||
gemm_workspace_.arguments.batch_count = problem_.batch_count;
|
||||
|
||||
@@ -270,17 +270,17 @@ int OperationProfiler::profile_all(
|
||||
ProblemSpace::Iterator problem_it = problem_space.begin();
|
||||
ProblemSpace::Iterator problem_end = problem_space.end();
|
||||
|
||||
bool continue_profiling = true, internal_error = false;
|
||||
bool continue_profiling = true;
|
||||
int retval = 0;
|
||||
|
||||
// For each problem in problem space
|
||||
for (; continue_profiling && problem_it != problem_end; ++problem_it) {
|
||||
|
||||
ProblemSpace::Problem problem = problem_it.at();
|
||||
|
||||
report.next_problem();
|
||||
|
||||
// For each operation in manifest
|
||||
for (auto const & operation_ptr : manifest) {
|
||||
int matched_operation_count = 0;
|
||||
for (auto const& operation_ptr : manifest) {
|
||||
|
||||
library::Operation const *operation = operation_ptr.get();
|
||||
|
||||
@@ -292,8 +292,8 @@ int OperationProfiler::profile_all(
|
||||
|
||||
// Execute compatible cutlass operations if they satisfy the current device's compute capability
|
||||
if (operation->description().kind == kind_ &&
|
||||
operation->description().provider == library::Provider::kCUTLASS &&
|
||||
options.device.compute_capability() >= min_cc &&
|
||||
operation->description().provider == library::Provider::kCUTLASS &&
|
||||
options.device.compute_capability() >= min_cc &&
|
||||
options.device.compute_capability() <= max_cc) {
|
||||
|
||||
std::string operation_name(operation->description().name);
|
||||
@@ -320,7 +320,10 @@ int OperationProfiler::profile_all(
|
||||
if (!filtered_by_name || !satisfies(operation->description(), problem_space, problem)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// we have found a kernel match, so increment the counter for match kernels
|
||||
++matched_operation_count;
|
||||
|
||||
// A. Initialize configuration
|
||||
Status status = this->initialize_configuration(
|
||||
options,
|
||||
@@ -374,7 +377,6 @@ int OperationProfiler::profile_all(
|
||||
//
|
||||
|
||||
// B. Verify CUTLASS
|
||||
|
||||
if (continue_profiling && options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
|
||||
|
||||
continue_profiling = this->verify_cutlass(
|
||||
@@ -426,10 +428,18 @@ int OperationProfiler::profile_all(
|
||||
if (!continue_profiling) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we did not find any kernels that match our filters and error_on_no_match was set, report an error
|
||||
if (options.profiling.error_on_no_match && matched_operation_count <= 0) {
|
||||
#if !NDEBUG
|
||||
std::cout << "Error: No matching kernels found with kernel selection filters [--error_on_no_match]" << std::endl;
|
||||
#endif
|
||||
retval = 1;
|
||||
}
|
||||
}
|
||||
|
||||
return internal_error ? 1 : 0;
|
||||
return retval;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -706,10 +706,12 @@ Options::Options(cutlass::CommandLine const &cmdline):
|
||||
}
|
||||
else if (cmdline.check_cmd_line_flag("kernels")) {
|
||||
cmdline.get_cmd_line_arguments("kernels", operation_names);
|
||||
profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
|
||||
}
|
||||
|
||||
if (cmdline.check_cmd_line_flag("ignore-kernels")) {
|
||||
cmdline.get_cmd_line_arguments("ignore-kernels", excluded_operation_names);
|
||||
profiling.error_on_no_match = cmdline.check_cmd_line_flag("error-on-no-match");
|
||||
}
|
||||
|
||||
// Prevent launches on the device for anything other than CUTLASS operation
|
||||
|
||||
@@ -196,6 +196,9 @@ public:
|
||||
/// If true, profiling is actually conducted.
|
||||
bool enabled;
|
||||
|
||||
/// If true, profiling returns an error code if no kernels are found to match the filters.
|
||||
bool error_on_no_match = false;
|
||||
|
||||
/// List of providers of each functionality to be profiled
|
||||
ProviderVector providers;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user