leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

v3.9 (#2185)

Browse Source 
* v3.8 update x

* fix blackwell gg

* doc change

* doc change

* doc change

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com>
This commit is contained in:
Yujia Zhai
2025-03-21 01:52:23 -04:00
committed by GitHub
co-authored by yuzhai Haicheng Wu Haicheng Wu
parent 8c4d1dc47d
commit 62750a2b75
334 changed files with 91517 additions and 2656 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/include/cutlass/library/arch_mappings.h
+5 -1
View File
@@ -127,9 +127,13 @@ template <typename OperatorClass> struct ArchMap<arch::Sm100, OperatorClass> {
template <> struct ArchMap<arch::Sm100, arch::OpClassTensorOp> {
static int const kMin = 100;
static int const kMax = 100;
static int const kMax = 101;
};
template <typename OperatorClass> struct ArchMap<arch::Sm120, OperatorClass> {
static int const kMin = 120;
static int const kMax = 120;
};
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/include/cutlass/library/library.h
+2 -3
View File
@@ -323,12 +323,11 @@ struct GemmUniversalArguments {
int swizzle_size{1};
int split_k_slices{1};
// For mixed input dtype kernels
bool is_mixed_dtype{false};
// For SM90 mixed input dtype kernels
bool is_sm90_mixed_dtype{false};
Sm90MixedInputWiderOperand wider_operand{Sm90MixedInputWiderOperand::B};
bool generate_scale_and_zero{false};
bool generate_dequantized_AB{false};
bool *dequantized_AB_ready{nullptr}; // Carry the info back to gemm_operation_profiler.cu
void *Scale{nullptr}; // Scale tensor
void *Zero{nullptr}; // Zero tensor
void *dequantized_AB{nullptr}; // Dequantized A or B tensor for verification
tools/library/src/block_scaled_gemm_operation_3x.hpp
+3 -3
View File
@@ -72,7 +72,7 @@ public:
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
using Sm100BlkScaledConfig = typename CollectiveMainloop::Sm100BlkScaledConfig;
using Sm1xxBlkScaledConfig = typename CollectiveMainloop::Sm1xxBlkScaledConfig;
static constexpr bool epilogue_scalefactor_generation = not cute::is_same_v<typename ThreadEpilogueOp::ElementBlockScaleFactor, void>;
static constexpr int32_t SFD_VectorSize = epilogue_scalefactor_generation ? ThreadEpilogueOp::SFVecSize : SFVecSize;
@@ -315,8 +315,8 @@ protected:
arguments->ldc, arguments->batch_stride_C);
operator_args.epilogue.dD = operator_args.epilogue.dC;
operator_args.mainloop.layout_SFA = Sm100BlkScaledConfig::tile_atom_to_shape_SFA(operator_args.problem_shape);
operator_args.mainloop.layout_SFB = Sm100BlkScaledConfig::tile_atom_to_shape_SFB(operator_args.problem_shape);
operator_args.mainloop.layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(operator_args.problem_shape);
operator_args.mainloop.layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(operator_args.problem_shape);
/* Query device SM count to pass onto the kernel as an argument, where needed */
operator_args.hw_info.sm_count = arguments->sm_count;
tools/library/src/gemm_operation_3x.hpp
+109 -96
View File
@@ -170,13 +170,12 @@ public:
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementA>();
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementB>();
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
@@ -252,13 +251,13 @@ protected:
};
template<template<int, class, class> class Policy, int Stages, class ClusterShape, class KernelSchedule>
static constexpr bool is_mixed_dtype_mainloop_(Policy<Stages, ClusterShape, KernelSchedule> policy) {
static constexpr bool is_sm90_mixed_dtype_mainloop_(Policy<Stages, ClusterShape, KernelSchedule> policy) {
return (cute::is_same_v<Policy<Stages, ClusterShape, KernelSchedule>,
cutlass::gemm::MainloopSm90TmaGmmaRmemAWarpSpecializedMixedInput<Stages, ClusterShape, KernelSchedule>>);
}
template <class DispatchPolicy>
static constexpr bool is_mixed_dtype_mainloop_(DispatchPolicy) {
static constexpr bool is_sm90_mixed_dtype_mainloop_(DispatchPolicy) {
return false;
}
@@ -380,7 +379,6 @@ protected:
arguments->batch_count);
// update arguments
if constexpr (IsRuntimeDataType) {
using ArrayElementA = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementA;
@@ -390,7 +388,7 @@ protected:
std::unordered_map<RuntimeDatatype, cute::UMMA::MXF8F6F4Format> mapping = {
{RuntimeDatatype::kE4M3, cute::UMMA::MXF8F6F4Format::E4M3},
{RuntimeDatatype::kE5M2, cute::UMMA::MXF8F6F4Format::E5M2},
{RuntimeDatatype::kE5M2, cute::UMMA::MXF8F6F4Format::E5M2},
{RuntimeDatatype::kE3M2, cute::UMMA::MXF8F6F4Format::E3M2},
{RuntimeDatatype::kE2M1, cute::UMMA::MXF8F6F4Format::E2M1}
};
@@ -412,10 +410,9 @@ protected:
}
else {
operator_args.mainloop.ptr_A = static_cast<ElementA const *>(arguments->A);
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
}
operator_args.mainloop.ptr_A = static_cast<ElementA const *>(arguments->A);
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
}
operator_args.epilogue.ptr_C = static_cast<ElementC const *>(arguments->C);
operator_args.epilogue.ptr_D = static_cast<ElementD *>(arguments->D);
@@ -441,11 +438,11 @@ protected:
operator_args.epilogue.dD = operator_args.epilogue.dC;
using MainloopPolicy = typename CollectiveMainloop::DispatchPolicy;
if constexpr(is_mixed_dtype_mainloop_(MainloopPolicy{})) {
int problem_m = arguments->problem_size.m();
int problem_n = arguments->problem_size.n();
int problem_k = arguments->problem_size.k();
int options_l = arguments->batch_count;
if constexpr(is_sm90_mixed_dtype_mainloop_(MainloopPolicy{})) {
const int problem_m = arguments->problem_size.m();
const int problem_n = arguments->problem_size.n();
const int problem_k = arguments->problem_size.k();
const int options_l = arguments->batch_count;
constexpr Sm90MixedInputWiderOperand wider_operand =
(cutlass::sizeof_bits<ElementA>::value > cutlass::sizeof_bits<ElementB>::value) ?
@@ -455,100 +452,118 @@ protected:
constexpr bool has_scale = !std::is_same_v<typename CollectiveMainloop::ElementScale, void>;
constexpr bool has_zero = !std::is_same_v<typename CollectiveMainloop::ElementZero, void>;
if constexpr(has_scale) {
int options_g = problem_k;
int scale_k = (problem_k + options_g - 1) / options_g;
constexpr bool is_A4B8 = (
cutlass::is_same_v<ElementA, cutlass::int4b_t> &&
(cutlass::is_same_v<ElementB, cutlass::float_e4m3_t> ||
cutlass::is_same_v<ElementB, cutlass::float_e5m2_t>));
constexpr bool is_A8B4 = (
cutlass::is_same_v<ElementB, cutlass::int4b_t> &&
(cutlass::is_same_v<ElementA, cutlass::float_e4m3_t> ||
cutlass::is_same_v<ElementA, cutlass::float_e5m2_t>));
constexpr bool is_int4_x_fp8 = is_A4B8 || is_A8B4;
const int options_g = problem_k;
const int scale_k = (problem_k + options_g - 1) / options_g;
// In int4 * fp8, ElementScale is a cutlass::Array, need to take out it's real element
using ElementScaleMainloop = typename CollectiveMainloop::ElementScale;
using ElementScale = typename UnderlyingElement<typename CollectiveMainloop::ElementScale>::type;
using StrideS = typename CollectiveMainloop::StrideScale;
// In ScaleOnly mode, we have allocated the same size of memory for arguments->Z and arguments->S
using ElementZero = std::conditional_t<
has_zero,
typename CollectiveMainloop::ElementZero,
ElementScale
>;
const int SZ_1st_dim = (wider_operand == Sm90MixedInputWiderOperand::A) ? problem_n : problem_m;
const size_t SZ_size = static_cast<size_t>(SZ_1st_dim * scale_k * options_l);
auto shape_SZ = cute::make_shape(SZ_1st_dim, scale_k, options_l);
ElementScale *ptr_S = static_cast<ElementScale *>(arguments->Scale);
ElementZero *ptr_Z = static_cast<ElementZero *>(arguments->Zero);
constexpr bool is_A4B8 = (
cutlass::is_same_v<ElementA, cutlass::int4b_t> &&
(cutlass::is_same_v<ElementB, cutlass::float_e4m3_t> ||
cutlass::is_same_v<ElementB, cutlass::float_e5m2_t>));
constexpr bool is_A8B4 = (
cutlass::is_same_v<ElementB, cutlass::int4b_t> &&
(cutlass::is_same_v<ElementA, cutlass::float_e4m3_t> ||
cutlass::is_same_v<ElementA, cutlass::float_e5m2_t>));
constexpr bool is_int4_x_fp8 = is_A4B8 || is_A8B4;
// 1. If arguments is initialized in profiler, S and Z needs to be allocated and filled
if (arguments->generate_scale_and_zero) {
// Need to fix max_dequant_val and min_dequant_val?
// If this is a convert-only kernel, we still need to generate dequantized A or B for verification,
// and in this case ElementScale is the same as ElementWide
// In int4 * fp8, ElementScale is a cutlass::Array, need to take out it's real element
using DummyElementScaleMainloop = std::conditional_t<
is_int4_x_fp8,
typename cutlass::Array<ElementWide, 8>,
ElementWide
>;
using ElementScaleMainloop = std::conditional_t<
has_scale,
typename CollectiveMainloop::ElementScale,
DummyElementScaleMainloop
>;
using ElementScale = std::conditional_t<
has_scale,
typename UnderlyingElement<typename CollectiveMainloop::ElementScale>::type,
ElementWide
>;
using StrideScale = typename CollectiveMainloop::StrideScale;
// In ScaleOnly mode, we have allocated the same size of memory for arguments->Z and arguments->S
using ElementZero = std::conditional_t<
has_zero,
typename CollectiveMainloop::ElementZero,
ElementScale
>;
const int SZ_1st_dim = (wider_operand == Sm90MixedInputWiderOperand::A) ? problem_n : problem_m;
const size_t SZ_size = static_cast<size_t>(SZ_1st_dim * scale_k * options_l);
auto shape_SZ = cute::make_shape(SZ_1st_dim, scale_k, options_l);
ElementScale *ptr_S = static_cast<ElementScale *>(arguments->Scale);
ElementZero *ptr_Z = static_cast<ElementZero *>(arguments->Zero);
// 1. If arguments is initialized in profiler, S and Z needs to be allocated and filled
if (arguments->generate_scale_and_zero) {
float scale_min = 1.0f, scale_max = 1.0f;
if constexpr(has_scale) {
const float elt_max_f = float(cutlass::platform::numeric_limits<ElementScale>::max());
// Need to fix max_dequant_val and min_dequant_val?
const float max_dequant_val = elt_max_f * 0.25f;
const float min_dequant_val = 0.5f;
const float scale_max = max_dequant_val / elt_max_f;
const float scale_min = min_dequant_val / elt_max_f;
uint64_t seed = 2023;
cutlass::reference::device::BlockFillRandomUniform(
ptr_S, SZ_size, seed, ElementScale(scale_max), ElementScale(scale_min));
scale_max = max_dequant_val / elt_max_f;
scale_min = min_dequant_val / elt_max_f;
}
uint64_t seed = 2023;
cutlass::reference::device::BlockFillRandomUniform(
ptr_S, SZ_size, seed, ElementScale(scale_max), ElementScale(scale_min));
// In ScaleOnly mode, set Z as zero for generating dequantized A or B
const float zero_max = has_zero ? 2.0f : 0.0f;
const float zero_min = has_zero ? -2.0f : 0.0f;
cutlass::reference::device::BlockFillRandomUniform(
ptr_Z, SZ_size, seed, ElementZero(zero_max), ElementZero(zero_min));
} // End of "if (arguments->generate_scale_and_zero)"
// In ScaleOnly mode, set Z as zero for generating dequantized A or B
const float zero_max = has_zero ? 2.0f : 0.0f;
const float zero_min = has_zero ? -2.0f : 0.0f;
cutlass::reference::device::BlockFillRandomUniform(
ptr_Z, SZ_size, seed, ElementZero(zero_max), ElementZero(zero_min));
} // End of "if (arguments->generate_scale_and_zero)"
// 2. Generate the dequantized A or B for verification
if (arguments->generate_dequantized_AB) {
StrideS stride_SZ = cutlass::make_cute_packed_stride(StrideS{}, shape_SZ);
auto layout_SZ = cute::make_layout(shape_SZ, stride_SZ);
if constexpr(wider_operand == Sm90MixedInputWiderOperand::A) {
if constexpr(is_StrideB_Layout) {
// The generator only generates row-major A and col-major B at the moment
// Need a way to read out the actual layout of B later
using ActualLayoutB = cutlass::layout::ColumnMajor;
using ActualStrideB = cutlass::detail::TagToStrideB_t<ActualLayoutB>;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideB, wider_operand, is_A8B4>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
else {
using ActualStrideB = typename CollectiveMainloop::StrideB;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideB, wider_operand, is_A8B4>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
// 2. Generate the dequantized A or B for verification
if (arguments->generate_dequantized_AB) {
StrideScale stride_SZ = cutlass::make_cute_packed_stride(StrideScale{}, shape_SZ);
auto layout_SZ = cute::make_layout(shape_SZ, stride_SZ);
if constexpr(wider_operand == Sm90MixedInputWiderOperand::A) {
if constexpr(is_StrideB_Layout) {
// The generator only generates row-major A and col-major B at the moment
// Need a way to read out the actual layout of B later
using ActualLayoutB = cutlass::layout::ColumnMajor;
using ActualStrideB = cutlass::detail::TagToStrideB_t<ActualLayoutB>;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideB, wider_operand, is_A8B4>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
else {
if constexpr(is_StrideA_Layout) {
// The generator only generates row-major A and col-major B at the moment
// Need a way to read out the actual layout of A later
using ActualLayoutA = cutlass::layout::RowMajor;
using ActualStrideA = cutlass::detail::TagToStrideA_t<ActualLayoutA>;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideA, wider_operand, is_A4B8>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
else {
using ActualStrideA = typename CollectiveMainloop::StrideA;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideA, wider_operand, is_A4B8>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
} // End of "if constexpr(wider_operand == Sm90MixedInputWiderOperand::A)"
arguments->dequantized_AB_ready[0] = true;
} // End of "if (arguments->generate_dequantized_AB)"
using ActualStrideB = typename CollectiveMainloop::StrideB;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideB, wider_operand, is_A8B4>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
}
else {
if constexpr(is_StrideA_Layout) {
// The generator only generates row-major A and col-major B at the moment
// Need a way to read out the actual layout of A later
using ActualLayoutA = cutlass::layout::RowMajor;
using ActualStrideA = cutlass::detail::TagToStrideA_t<ActualLayoutA>;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideA, wider_operand, is_A4B8>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
else {
using ActualStrideA = typename CollectiveMainloop::StrideA;
dequantize_encode_<ElementWide, ElementNarrow, ElementScaleMainloop, ActualStrideA, wider_operand, is_A4B8>(
operator_args, arguments, stream, problem_m, problem_k, options_l, options_g, ptr_S, ptr_Z, SZ_size, layout_SZ);
}
} // End of "if constexpr(wider_operand == Sm90MixedInputWiderOperand::A)"
} // End of "if (arguments->generate_dequantized_AB)"
// 3. Put arguments in mainloop
// 3. Put Scale and Zero in mainloop
if constexpr(has_scale) {
if constexpr(is_int4_x_fp8) {
operator_args.mainloop.ptr_S = static_cast<ElementScaleMainloop const*>(arguments->packed_Scale);
}
else {
operator_args.mainloop.ptr_S = static_cast<ElementScale const*>(arguments->Scale);
}
operator_args.mainloop.dS = cutlass::make_cute_packed_stride(StrideS{}, shape_SZ);
operator_args.mainloop.dS = cutlass::make_cute_packed_stride(StrideScale{}, shape_SZ);
operator_args.mainloop.group_size = options_g;
if constexpr(has_zero) {
operator_args.mainloop.ptr_Z = static_cast<ElementZero const*>(arguments->Zero);
@@ -580,7 +595,7 @@ protected:
handle_shuffle_tensor_<ElementA, ActualStrideA, LayoutA_Reordered, LayoutAtomQuant, wider_operand>(
operator_args, arguments, problem_m, problem_k, options_l);
}
} // End of "if constexpr(is_mixed_dtype_mainloop_(MainloopPolicy{}))"
} // End of "if constexpr(is_sm90_mixed_dtype_mainloop_(MainloopPolicy{}))"
/* Query device SM count and max active clusters to pass onto the kernel as an argument, where needed */
operator_args.hw_info.sm_count = arguments->sm_count;
@@ -609,7 +624,6 @@ protected:
operator_args.scheduler.splits = arguments->split_k_slices;
}
if constexpr (Operator::ArchTag::kMinComputeCapability >= 100) {
operator_args.hw_info.cluster_shape = dim3(
arguments->cluster_shape.m(),
@@ -620,7 +634,6 @@ protected:
arguments->cluster_shape_fallback.n(),
arguments->cluster_shape_fallback.k());
}
return status;
}
@@ -632,7 +645,7 @@ public:
GemmUniversalArguments const *arguments =
static_cast<GemmUniversalArguments const *>(arguments_ptr);
OperatorArguments args;
auto status = update_arguments_(args, arguments);
if (status != Status::kSuccess) {
return status;
tools/library/src/grouped_gemm_operation_3x.hpp
+2 -2
View File
@@ -512,8 +512,8 @@ public:
auto N = get<1>(shape);
auto K = get<2>(shape);
auto layout_SFA = CollectiveMainloop::Sm100BlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(M, N, K, 1));
auto layout_SFB = CollectiveMainloop::Sm100BlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, 1));
auto layout_SFA = CollectiveMainloop::Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(M, N, K, 1));
auto layout_SFB = CollectiveMainloop::Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(M, N, K, 1));
layout_SFA_host[group_idx] = layout_SFA;
layout_SFB_host[group_idx] = layout_SFB;
}
tools/library/src/reference/block_scaled_gemm_reference_operation.h
+5 -5
View File
@@ -243,14 +243,14 @@ public:
auto stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
auto stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
using Sm100BlockScaledConfig = cutlass::detail::Sm100BlockScaledConfig<SFVecSize>;
using Sm1xxBlockScaledConfig = cutlass::detail::Sm1xxBlockScaledConfig<SFVecSize>;
auto A = cute::make_tensor(detail::make_iterator(static_cast<ElementA const*>(args.A)),
cute::make_layout(cute::make_shape(M, K, L), stride_a));
auto SfA = make_tensor(static_cast<ElementSFA const*>(args.SFA), Sm100BlockScaledConfig::tile_atom_to_shape_SFA(problem_shape_MNKL));
auto SfA = make_tensor(static_cast<ElementSFA const*>(args.SFA), Sm1xxBlockScaledConfig::tile_atom_to_shape_SFA(problem_shape_MNKL));
auto B = cute::make_tensor(detail::make_iterator(static_cast<ElementB const*>(args.B)),
cute::make_layout(cute::make_shape(N, K, L), stride_b));
auto SfB = make_tensor(static_cast<ElementSFB const*>(args.SFB), Sm100BlockScaledConfig::tile_atom_to_shape_SFB(problem_shape_MNKL));
auto SfB = make_tensor(static_cast<ElementSFB const*>(args.SFB), Sm1xxBlockScaledConfig::tile_atom_to_shape_SFB(problem_shape_MNKL));
auto C = [&]() {
if constexpr (not is_same_v<ElementC, void>) {
@@ -273,11 +273,11 @@ public:
if constexpr (not is_same_v<ElementSFD, void>) {
using Sm100BlockScaledOutputConfig = cutlass::detail::Sm100BlockScaledOutputConfig<
using Sm1xxBlockScaledOutputConfig= cutlass::detail::Sm1xxBlockScaledOutputConfig<
EpilogueSFVecSize
>;
auto SfD = cute::make_tensor(detail::make_iterator(static_cast<ElementSFD*>(args.SFD)), Sm100BlockScaledOutputConfig::tile_atom_to_shape_SFD(problem_shape_MNKL));
auto SfD = cute::make_tensor(detail::make_iterator(static_cast<ElementSFD*>(args.SFD)), Sm1xxBlockScaledOutputConfig::tile_atom_to_shape_SFD(problem_shape_MNKL));
cutlass::reference::host::GettBlockScalingEpilogueParams<
ElementCompute, ElementAccumulator, ElementCompute,
tools/library/src/sparse_gemm_operation_3x.hpp
+66 -5
View File
@@ -36,13 +36,22 @@
#include "cutlass/cutlass.h"
#include "cutlass/detail/collective.hpp"
#include "cutlass/array.h"
#include "cutlass/array_subbyte.h"
#include "cutlass/library/library.h"
#include "cutlass/library/util.h"
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp" // StructuredSparseCompressor
#include "cutlass/transform/device/transform_universal_adapter.hpp" // TransformUniversalAdapter
#include "cutlass/util/packed_stride.hpp" // make_cute_packed_stride
#include "gemm_operation_3x.hpp"
#include "library_internal.h"
#include "cutlass/gemm/dispatch_policy.hpp"
#include "cutlass/util/packed_stride.hpp"
#include "cutlass/util/mixed_dtype_utils.hpp"
#include "cutlass/util/device_memory.h"
#include "cutlass/util/reference/device/tensor_fill.h"
#include "cutlass/util/reference/device/tensor_compare.h"
#include "cute/tensor.hpp"
#include <unordered_map>
///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -77,6 +86,16 @@ public:
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementA>();
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementB>();
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
using ElementE = typename CollectiveMainloop::ElementE;
using LayoutE = typename CollectiveMainloop::LayoutE;
using SparseConfig = typename CollectiveMainloop::SparseConfig;
@@ -170,9 +189,41 @@ protected:
arguments->batch_count);
// update arguments
operator_args.mainloop.ptr_A = reinterpret_cast<ElementA const *>(device_a_compressed_ptr);
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
operator_args.mainloop.ptr_E = reinterpret_cast<ElementE const *>(device_e_ptr);
if constexpr (IsRuntimeDataType) {
using ArrayElementA = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementA;
using ArrayElementB = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementB;
operator_args.mainloop.ptr_A = static_cast<ArrayElementA const *>(device_a_compressed_ptr);
operator_args.mainloop.ptr_B = static_cast<ArrayElementB const *>(arguments->B);
std::unordered_map<RuntimeDatatype, cute::UMMA::MXF8F6F4Format> mapping = {
{RuntimeDatatype::kE4M3, cute::UMMA::MXF8F6F4Format::E4M3},
{RuntimeDatatype::kE5M2, cute::UMMA::MXF8F6F4Format::E5M2},
{RuntimeDatatype::kE3M2, cute::UMMA::MXF8F6F4Format::E3M2},
{RuntimeDatatype::kE2M1, cute::UMMA::MXF8F6F4Format::E2M1}
};
auto iter_runtime_a = mapping.find(arguments->runtime_input_datatype_a);
auto iter_runtime_b = mapping.find(arguments->runtime_input_datatype_b);
if (iter_runtime_a != mapping.end()) {
operator_args.mainloop.runtime_data_type_a = iter_runtime_a->second;
} else {
assert("invalid runtime argument for datatype A!");
}
if (iter_runtime_b != mapping.end()) {
operator_args.mainloop.runtime_data_type_b = iter_runtime_b->second;
} else {
assert("invalid runtime argument for datatype B!");
}
}
else {
operator_args.mainloop.ptr_A = static_cast<ElementA const *>(device_a_compressed_ptr);
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
}
operator_args.mainloop.ptr_E = static_cast<ElementE const *>(device_e_ptr);
operator_args.epilogue.ptr_C = static_cast<ElementC const *>(arguments->C);
operator_args.epilogue.ptr_D = static_cast<ElementD *>(arguments->D);
@@ -184,7 +235,7 @@ protected:
arguments->ldc, arguments->batch_stride_C);
operator_args.epilogue.dD = operator_args.epilogue.dC;
/* Query device SM count to pass onto the kernel as an argument, where needed */
/* Query device SM count and max active clusters to pass onto the kernel as an argument, where needed */
operator_args.hw_info.sm_count = arguments->sm_count;
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.max_swizzle_size)>) {
operator_args.scheduler.max_swizzle_size = arguments->swizzle_size;
@@ -208,6 +259,16 @@ protected:
operator_args.scheduler.splits = arguments->split_k_slices;
}
if constexpr (Operator::ArchTag::kMinComputeCapability >= 100) {
operator_args.hw_info.cluster_shape = dim3(
arguments->cluster_shape.m(),
arguments->cluster_shape.n(),
arguments->cluster_shape.k());
operator_args.hw_info.cluster_shape_fallback = dim3(
arguments->cluster_shape_fallback.m(),
arguments->cluster_shape_fallback.n(),
arguments->cluster_shape_fallback.k());
}
return status;
}
tools/profiler/include/cutlass/profiler/block_scaled_gemm_operation_profiler.h
+38
View File
@@ -37,6 +37,7 @@
#pragma once
#include <vector>
#include <array>
#include <string>
#include <memory>
#include <algorithm>
@@ -71,6 +72,14 @@ public:
cutlass::library::GemmUniversalMode mode{library::GemmUniversalMode::kGemm};
/// For profiling purposes
std::vector<gemm::GemmCoord> problem_sizes;
std::vector<std::array<int64_t, 3>> leading_dims;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
std::vector<cutlass::library::RasterOrder> raster_orders;
std::vector<int> swizzle_sizes;
int64_t m{16};
int64_t n{16};
int64_t k{16};
@@ -119,6 +128,14 @@ public:
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
int64_t bytes_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const;
int64_t flops_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const;
/// Total number of bytes loaded
int64_t bytes(library::BlockScaledGemmDescription const &operation_desc) const;
@@ -239,6 +256,27 @@ public:
protected:
/// Update workspace configuration according to flexible user setups
void update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size);
/// Update performance result configuration according to flexible user setups
void update_result_(
PerformanceResult &result,
library::BlockScaledGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size);
/// Initializes the performance result
void initialize_result_(
PerformanceResult &result,
tools/profiler/include/cutlass/profiler/gemm_operation_profiler.h
+37
View File
@@ -35,6 +35,7 @@
#pragma once
#include <vector>
#include <array>
#include <string>
#include <memory>
#include <algorithm>
@@ -69,6 +70,14 @@ public:
cutlass::library::GemmUniversalMode mode{library::GemmUniversalMode::kGemm};
/// For profiling purposes
std::vector<gemm::GemmCoord> problem_sizes;
std::vector<std::array<int64_t, 3>> leading_dims;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
std::vector<cutlass::library::RasterOrder> raster_orders;
std::vector<int> swizzle_sizes;
int64_t m{16};
int64_t n{16};
int64_t k{16};
@@ -120,6 +129,14 @@ public:
ProblemSpace const &problem_space,
ProblemSpace::Problem const &problem);
int64_t bytes_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const;
int64_t flops_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const;
/// Total number of bytes loaded
int64_t bytes(library::GemmDescription const &operation_desc) const;
@@ -243,6 +260,26 @@ public:
ProblemSpace::Problem const &problem);
protected:
/// Update workspace configuration according to flexible user setups
void update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size);
/// Update performance result configuration according to flexible user setups
void update_result_(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size);
/// Initializes the performance result
void initialize_result_(
tools/profiler/include/cutlass/profiler/options.h
+20 -2
View File
@@ -208,8 +208,23 @@ public:
/// Minimum number of iterations to profile
int min_iterations{10};
/// If true, profiling with cuda graph enabled.
bool use_cuda_graphs{false};
/// If enabled, the CUTLASS profiler searches for the best-performing kernel
/// within the subset of kernels matching a kernel filter regex. The best
/// performance is determined by screening over a set of predefined M/N/K
/// sizes and performance-related parameters, including cluster shapes,
/// swizzle sizes, and rasterization orders.
/// For now, it only supports legacy GEMM and blockscaled GEMM.
bool enable_kernel_performance_search{false};
/// If enabled, the CUTLASS profiler searches for the best-performing kernel
/// for a given M/N/K problem size by evaluating various performance-related
/// parameters such as cluster shapes, swizzle sizes, and rasterization orders.
/// For now, it only supports legacy GEMM and blockscaled GEMM.
bool enable_best_kernel_for_fixed_shape{false};
/// Number of ms to sleep between profiling periods (ms)
int sleep_duration{50};
@@ -264,8 +279,11 @@ public:
/// Prints human-readable text to stdout. If false, nothing is written to stdout
bool verbose;
/// Sort results by (currently by flops-per-byte)
bool sort_results;
/// Sort results by flops-per-byte
bool sort_flops_per_byte;
/// Sort results by flops-per-second
bool sort_flops_per_sec;
/// Prints the name of the kernel being profiled before running the kernel.
/// This is useful for determining which kernel is causing a run of the profiler to hang
tools/profiler/include/cutlass/profiler/performance_report.h
+2 -1
View File
@@ -92,7 +92,8 @@ public:
void next_problem();
void append_result(PerformanceResult result);
void sort_results(PerformanceResultVector &results);
void sort_flops_per_byte(PerformanceResultVector &results);
void sort_flops_per_sec(PerformanceResultVector &results);
void append_results(PerformanceResultVector const &results);
public:
tools/profiler/include/cutlass/profiler/performance_result.h
+6
View File
@@ -105,6 +105,12 @@ struct PerformanceResult {
runtime(0)
{ }
// Copy constructor for deep copy
PerformanceResult(const PerformanceResult& other) = default;
// Explicitly define copy assignment operator
PerformanceResult& operator=(const PerformanceResult& other) = default;
/// Returns true if the runtime is valid
bool good() const {
return runtime > 0;
tools/profiler/src/block_scaled_gemm_operation_profiler.cu
+329 -61
View File
@@ -174,7 +174,6 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
@@ -204,7 +203,6 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
// default value
this->cluster_k_fallback = 0;
}
if (!arg_as_SplitKModeID(this->split_k_mode, "split_k_mode", problem_space, problem)) {
// default value
@@ -314,23 +312,61 @@ Status BlockScaledGemmOperationProfiler::GemmProblem::parse(
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
// instantiation for deep profiling
int num_sizes = 8;
this->problem_sizes.resize(num_sizes);
this->leading_dims.resize(num_sizes, {0, 0, 0});
int m0 = 1024;
int n0 = 1024;
int k0 = 1024;
for (int i = 0; i < num_sizes; i++) {
auto m = m0 * (i + 3);
auto n = n0 * (i + 3);
auto k = k0 * (i + 3);
this->problem_sizes[i] = {m, n, k};
this->leading_dims[i] = {
DeviceAllocation::get_packed_layout(operation_desc.A.layout, {int(m), int(k)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.B.layout, {int(k), int(n)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.C.layout, {int(m), int(n)}).front()
};
}
this->raster_orders = {
cutlass::library::RasterOrder::kAlongN,
cutlass::library::RasterOrder::kAlongM
};
this->swizzle_sizes = {1, 2, 4, 8};
this->preferred_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}, {4, 1, 1}, {4, 2, 1}, {4, 4, 1}, {8, 2, 1}
};
this->fallback_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}
};
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScaledGemmDescription const &operation_desc) const {
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_shape.m() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_shape.n() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
}
bytes *= batch_count;
@@ -338,9 +374,16 @@ int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScale
return bytes;
}
int64_t BlockScaledGemmOperationProfiler::GemmProblem::bytes(library::BlockScaledGemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return bytes_with_problem_shape(operation_desc, problem_shape);
}
/// Total number of flops computed
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScaledGemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops_with_problem_shape(
library::BlockScaledGemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t flops_ = (int64_t(problem_shape.m()) * problem_shape.n() * problem_shape.k() + problem_shape.m() * problem_shape.n()) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
@@ -362,6 +405,10 @@ int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScale
return flops_;
}
int64_t BlockScaledGemmOperationProfiler::GemmProblem::flops(library::BlockScaledGemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return flops_with_problem_shape(operation_desc, problem_shape);
}
/// Initializes a performance result
void BlockScaledGemmOperationProfiler::GemmProblem::initialize_result(
@@ -480,8 +527,8 @@ Status BlockScaledGemmOperationProfiler::initialize_configuration(
gemm_workspace_.arguments.swizzle_size = problem_.swizzle_size;
gemm_workspace_.arguments.raster_order = problem_.raster_order;
gemm_workspace_.arguments.norm_constant = 0;
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.split_k_slices = problem_.split_k_slices;
@@ -499,8 +546,74 @@ Status BlockScaledGemmOperationProfiler::initialize_configuration(
}
initialize_result_(this->model_result_, options, operation_desc, problem_space);
return operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
}
void BlockScaledGemmOperationProfiler::update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
gemm_workspace.arguments.problem_size.n() = problem_shape.n();
gemm_workspace.arguments.problem_size.k() = problem_shape.k();
gemm_workspace.arguments.lda = leading_dim[0];
gemm_workspace.arguments.ldb = leading_dim[1];
gemm_workspace.arguments.ldc = leading_dim[2];
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
gemm_workspace.configuration.ldc = leading_dim[2];
}
void BlockScaledGemmOperationProfiler::update_result_(
PerformanceResult &result,
library::BlockScaledGemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
set_argument(result, "m", problem_space, problem_shape.m());
set_argument(result, "n", problem_space, problem_shape.n());
set_argument(result, "k", problem_space, problem_shape.k());
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
/// Initializes the performance result
@@ -619,27 +732,38 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
bool enable_deep_profiling = options.profiling.enable_kernel_performance_search;
int seed_shift = 0;
// When exhaustive performance search (deep profiling) option is enabled, device buffers are initialized to the largest problem shape
// so that later performance search can re-use those buffers.
int init_m = enable_deep_profiling ? std::max(int(problem_.m), problem_.problem_sizes.back().m()) : int(problem_.m);
int init_n = enable_deep_profiling ? std::max(int(problem_.n), problem_.problem_sizes.back().n()) : int(problem_.n);
int init_k = enable_deep_profiling ? std::max(int(problem_.k), problem_.problem_sizes.back().k()) : int(problem_.k);
int init_lda = enable_deep_profiling ? int(std::max(problem_.lda, problem_.leading_dims.back()[0])) : int(problem_.lda);
int init_ldb = enable_deep_profiling ? int(std::max(problem_.ldb, problem_.leading_dims.back()[1])) : int(problem_.ldb);
int init_ldc = enable_deep_profiling ? int(std::max(problem_.ldc, problem_.leading_dims.back()[2])) : int(problem_.ldc);
gemm_workspace_.A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
{init_m, init_k},
{init_lda},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
);
int sfa_m = round_up(int(problem_.m), 128);
int sfb_n = round_up(int(problem_.n), 128);
int sfa_sfb_k = round_up(ceil_div(int(problem_.k), operation_desc.SFVecSize), 4);
int sfa_m = round_up(init_m, 128);
int sfb_n = round_up(init_n, 128);
int sfa_sfb_k = round_up(ceil_div(init_k, operation_desc.SFVecSize), 4);
int sfd_m = operation_desc.SFD.layout == cutlass::library::LayoutTypeID::kRowMajor ?
sfa_m : round_up(ceil_div(int(problem_.m), operation_desc.EpilogueSFVecSize), 4);
sfa_m : round_up(ceil_div(init_m, operation_desc.EpilogueSFVecSize), 4);
int sfd_n = operation_desc.SFD.layout == cutlass::library::LayoutTypeID::kRowMajor ?
round_up(ceil_div(int(problem_.n), operation_desc.EpilogueSFVecSize), 4) : sfb_n;
round_up(ceil_div(init_n, operation_desc.EpilogueSFVecSize), 4) : sfb_n;
gemm_workspace_.SFA = device_context.allocate_and_initialize_tensor(
@@ -671,8 +795,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
{init_k, init_n},
{init_ldb},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
@@ -683,8 +807,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"C",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
seed_shift++,
0 // device_index
@@ -695,8 +819,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
@@ -706,8 +830,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_.problem_count,
0 // device_index
);
@@ -766,8 +890,8 @@ Status BlockScaledGemmOperationProfiler::initialize_workspace(
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_.arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_.arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_.arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_.arguments.batch_count = problem_.batch_count;
gemm_workspace_.arguments.lda = problem_.lda;
@@ -1247,42 +1371,186 @@ bool BlockScaledGemmOperationProfiler::profile(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
if (options.profiling.enable_kernel_performance_search || options.profiling.enable_best_kernel_for_fixed_shape) {
library::BlockScaledGemmDescription const &operation_desc =
static_cast<library::BlockScaledGemmDescription const &>(operation->description());
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
PerformanceResult const &result,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size) -> std::optional<PerformanceResult> {
// Initialize structure containing GEMM arguments
gemm_workspace_.arguments.A = gemm_workspace_.A->data();
gemm_workspace_.arguments.B = gemm_workspace_.B->data();
gemm_workspace_.arguments.SFA = gemm_workspace_.SFA->data();
gemm_workspace_.arguments.SFB = gemm_workspace_.SFB->data();
gemm_workspace_.arguments.C = gemm_workspace_.C->data();
gemm_workspace_.arguments.D = gemm_workspace_.Computed->data();
gemm_workspace_.arguments.alpha = problem_.alpha.data();
gemm_workspace_.arguments.beta = problem_.beta.data();
gemm_workspace_.arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_.arguments.batch_stride_A = gemm_workspace_.A->batch_stride();
gemm_workspace_.arguments.batch_stride_B = gemm_workspace_.B->batch_stride();
gemm_workspace_.arguments.batch_stride_C = gemm_workspace_.C->batch_stride();
gemm_workspace_.arguments.batch_stride_D = gemm_workspace_.Computed->batch_stride();
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_.arguments.D = gemm_workspace_.device_workspace.data();
gemm_workspace_.arguments.alpha = problem_.alpha_one.data();
gemm_workspace_.arguments.beta = problem_.beta_zero.data();
gemm_workspace_.reduction_arguments.workspace = gemm_workspace_.device_workspace.data();
gemm_workspace_.reduction_arguments.source = gemm_workspace_.C->data();
gemm_workspace_.reduction_arguments.destination = gemm_workspace_.Computed->data();
gemm_workspace_.reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_.reduction_arguments.beta = problem_.beta.data();
gemm_workspace_.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(gemm_workspace_, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
const auto can_implement = operation->can_implement(&gemm_workspace_.configuration, &gemm_workspace_.arguments);
if (can_implement != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
library::Operation const* underlying_operation = operation;
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&gemm_workspace_.configuration);
gemm_workspace_.host_workspace.resize(workspace_size, 0);
workspace_size = underlying_operation->get_device_workspace_size(&gemm_workspace_.configuration,
&gemm_workspace_.arguments);
gemm_workspace_.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
Status status = Status::kSuccess;
status = underlying_operation->initialize(
&gemm_workspace_.configuration,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data());
if (status != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
curr_result.status = profile_cutlass_(
curr_result,
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
return curr_result;
};
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
for (size_t i = 0; i < 3; ++i) {
if (preferred_cluster[i] % fallback_cluster[i] != 0) {
return false;
}
}
return true;
};
// Helper function to select the best performance number among a list.
auto select_best_candidate = [&](std::vector<PerformanceResult> &candidates) {
assert(!candidates.empty() && "Candidates vector should not be empty");
auto best_iter = std::max_element(
candidates.begin(), candidates.end(),
[](PerformanceResult const &a, PerformanceResult const &b) {
return a.gflops_per_sec() < b.gflops_per_sec();
}
);
assert(best_iter != candidates.end() && "No candidate found despite non-empty candidates vector");
results_.push_back(std::move(*best_iter));
};
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
}
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
// With the fixed shape option turned on, only a specific problem shape is tested.
if (options.profiling.enable_best_kernel_for_fixed_shape) {
this->problem_.problem_sizes = {{int(this->problem_.m), int(this->problem_.n), int(this->problem_.k)}};
this->problem_.leading_dims = {{this->problem_.lda, this->problem_.ldb, this->problem_.ldc}};
}
for (int i = 0; i < int(this->problem_.problem_sizes.size()); i++) {
gemm::GemmCoord problem_shape = problem_.problem_sizes[i];
std::array<int64_t, 3> leading_dim = problem_.leading_dims[i];
auto result_opt = initialize_and_profile(result_base, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
if (result_opt) { // Only add valid results
candidates.push_back(*result_opt);
}
}
}// for raster_order
}// for swizzle_size
}// for fallback_cluster
}// for swizzle_size
if (candidates.empty()) {
return false;
}
select_best_candidate(candidates);
}
else {
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
&gemm_workspace_.arguments,
gemm_workspace_.host_workspace.data(),
gemm_workspace_.device_workspace.data()
);
}
return true;
}
tools/profiler/src/gemm_operation_profiler.cu
+363 -66
View File
@@ -53,7 +53,6 @@
namespace cutlass {
namespace profiler {
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Ctor
@@ -175,7 +174,6 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->k = 1024;
}
if (!arg_as_int(this->cluster_m, "cluster_m", problem_space, problem)) {
// default value
this->cluster_m = 1;
@@ -205,7 +203,6 @@ Status GemmOperationProfiler::GemmProblem::parse(
// default value
this->cluster_k_fallback = 0;
}
if (!arg_as_bool(this->use_pdl, "use_pdl", problem_space, problem)) {
// default value
@@ -315,33 +312,80 @@ Status GemmOperationProfiler::GemmProblem::parse(
this->ldc = DeviceAllocation::get_packed_layout(
operation_desc.C.layout, {int(this->m), int(this->n)}).front();
// instantiation
int num_sizes = 8;
this->problem_sizes.resize(num_sizes);
this->leading_dims.resize(num_sizes, {0, 0, 0});
int m0 = 1024;
int n0 = 1024;
int k0 = 1024;
for (int i = 0; i < num_sizes; i++) {
auto m = m0 * (i + 1);
auto n = n0 * (i + 1);
auto k = k0 * (i + 1);
this->problem_sizes[i] = {m, n, k};
this->leading_dims[i] = {
DeviceAllocation::get_packed_layout(operation_desc.A.layout, {int(m), int(k)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.B.layout, {int(k), int(n)}).front(),
DeviceAllocation::get_packed_layout(operation_desc.C.layout, {int(m), int(n)}).front()
};
}
this->raster_orders = {
cutlass::library::RasterOrder::kAlongN,
cutlass::library::RasterOrder::kAlongM
};
this->swizzle_sizes = {1, 2, 4, 8};
this->preferred_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}, {4, 1, 1}, {4, 2, 1}, {4, 4, 1}, {8, 2, 1}
};
this->fallback_clusters = {
{1, 1, 1}, {2, 1, 1}, {2, 2, 1}
};
return Status::kSuccess;
}
/// Total number of bytes loaded
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
int64_t GemmOperationProfiler::GemmProblem::bytes_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
// Input bytes read and Output bytes written for the gemm problem
int64_t bytes =
int64_t(library::sizeof_bits(operation_desc.A.element) * m / 8) * k +
int64_t(library::sizeof_bits(operation_desc.B.element) * n / 8) * k +
int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
int64_t(library::sizeof_bits(operation_desc.A.element) * problem_shape.m() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.B.element) * problem_shape.n() / 8) * problem_shape.k() +
int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
// Set is_beta_zero true if beta is zero
bool is_beta_zero = std::all_of(beta.begin(), beta.end(), [](uint8_t i) { return i==0; });
// Output bytes read for the gemm problem for non-zero beta values
if (!is_beta_zero) {
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * m / 8) * n;
bytes += int64_t(library::sizeof_bits(operation_desc.C.element) * problem_shape.m() / 8) * problem_shape.n();
}
bytes *= batch_count;
return bytes;
}
/// Total number of bytes loaded
int64_t GemmOperationProfiler::GemmProblem::bytes(library::GemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return bytes_with_problem_shape(operation_desc, problem_shape);
}
/// Total number of flops computed
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
int64_t flops_ = (int64_t(m) * n * k + m * n) * 2 * batch_count;
int64_t GemmOperationProfiler::GemmProblem::flops_with_problem_shape(
library::GemmDescription const &operation_desc,
gemm::GemmCoord const &problem_shape) const {
int64_t flops_ = (int64_t(problem_shape.m()) * problem_shape.n() * problem_shape.k() + problem_shape.m() * problem_shape.n()) * 2 * batch_count;
// complex-valued support
switch (operation_desc.tile_description.math_instruction.math_operation) {
@@ -363,6 +407,12 @@ int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const
return flops_;
}
/// Total number of flops computed
int64_t GemmOperationProfiler::GemmProblem::flops(library::GemmDescription const &operation_desc) const {
gemm::GemmCoord problem_shape({int(m), int(n), int(k)});
return flops_with_problem_shape(operation_desc, problem_shape);
}
/// Initializes a performance result
void GemmOperationProfiler::GemmProblem::initialize_result(
@@ -440,8 +490,9 @@ Status GemmOperationProfiler::initialize_configuration(
Status status = problem_.parse(operation_desc, problem_space, problem);
// Note: this is a temporary workaround
bool is_current_operation_sm90_mixed_dtype_shuffle = (strstr(operation_desc.name, "_shfl") != NULL);
if (is_current_operation_sm90_mixed_dtype_shuffle && (problem_.enable_sm90_mixed_dtype_shuffle_test == false)) {
bool is_sm90_operation = (strstr(operation_desc.name, "_sm90") != NULL);
bool is_sm90_mixed_dtype_shuffle_operation = (strstr(operation_desc.name, "_shfl") != NULL);
if (is_sm90_mixed_dtype_shuffle_operation && (problem_.enable_sm90_mixed_dtype_shuffle_test == false)) {
return Status::kErrorInvalidProblem;
}
@@ -457,7 +508,7 @@ Status GemmOperationProfiler::initialize_configuration(
library::NumericTypeID b_elem = library::get_real_type(operation_desc.B.element);
int a_elem_bits = library::sizeof_bits(a_elem);
int b_elem_bits = library::sizeof_bits(b_elem);
bool is_mixed_input = (a_elem_bits != b_elem_bits);
bool is_sm90_mixed_dtype_operation = is_sm90_operation && (a_elem_bits != b_elem_bits);
for (size_t i = 0; i < device_count; ++i) {
cudaSetDevice(options.device.device_id(i));
@@ -519,7 +570,7 @@ Status GemmOperationProfiler::initialize_configuration(
initialize_result_(this->model_result_, options, operation_desc, problem_space);
if (is_mixed_input)
if (is_sm90_mixed_dtype_operation)
{
const int options_g = problem_.k;
const int options_l = problem_.batch_count;
@@ -577,16 +628,14 @@ Status GemmOperationProfiler::initialize_configuration(
// Here is the first touch of the arguments, mark the mixed dtype,
// populate the scale and zero tensors in the following can_implement() call later.
// A and B are not populated at this moment, so do not update the dequantized A or B
gemm_workspace_[i].arguments.is_mixed_dtype = true;
gemm_workspace_[i].arguments.is_sm90_mixed_dtype = true;
gemm_workspace_[i].arguments.wider_operand = (a_elem_bits > b_elem_bits) ? cutlass::library::Sm90MixedInputWiderOperand::A : cutlass::library::Sm90MixedInputWiderOperand::B;
gemm_workspace_[i].arguments.generate_scale_and_zero = true;
gemm_workspace_[i].arguments.generate_dequantized_AB = false;
gemm_workspace_[i].arguments.dequantized_AB_ready = (bool *) malloc(sizeof(bool));
gemm_workspace_[i].arguments.dequantized_AB_ready[0] = false;
gemm_workspace_[i].arguments.Scale = gemm_workspace_[i].Scale->data();
gemm_workspace_[i].arguments.Zero = gemm_workspace_[i].Zero->data();
gemm_workspace_[i].arguments.packed_Scale = gemm_workspace_[i].packed_Scale->data();
} // End of "if (is_mixed_input)"
} // End of "if (is_sm90_mixed_dtype_operation)"
const auto can_implement = operation->can_implement(&gemm_workspace_[i].configuration, &gemm_workspace_[i].arguments);
if (can_implement != Status::kSuccess) {
@@ -604,6 +653,72 @@ Status GemmOperationProfiler::initialize_configuration(
return status;
}
void GemmOperationProfiler::update_workspace_(
GemmWorkspace &gemm_workspace,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size
) {
gemm_workspace.arguments.problem_size.m() = problem_shape.m();
gemm_workspace.arguments.problem_size.n() = problem_shape.n();
gemm_workspace.arguments.problem_size.k() = problem_shape.k();
gemm_workspace.arguments.lda = leading_dim[0];
gemm_workspace.arguments.ldb = leading_dim[1];
gemm_workspace.arguments.ldc = leading_dim[2];
gemm_workspace.arguments.swizzle_size = swizzle_size;
gemm_workspace.arguments.raster_order = raster_order;
gemm_workspace.arguments.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.arguments.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.problem_size.m() = problem_shape.m();
gemm_workspace.configuration.problem_size.n() = problem_shape.n();
gemm_workspace.configuration.problem_size.k() = problem_shape.k();
gemm_workspace.configuration.cluster_shape = {int(preferred_cluster[0]), int(preferred_cluster[1]), int(preferred_cluster[2])};
gemm_workspace.configuration.cluster_shape_fallback = {int(fallback_cluster[0]), int(fallback_cluster[1]), int(fallback_cluster[2])};
gemm_workspace.configuration.lda = leading_dim[0];
gemm_workspace.configuration.ldb = leading_dim[1];
gemm_workspace.configuration.ldc = leading_dim[2];
}
void GemmOperationProfiler::update_result_(
PerformanceResult &result,
library::GemmDescription const &operation_desc,
ProblemSpace const &problem_space,
gemm::GemmCoord const &problem_shape,
cutlass::library::RasterOrder const &raster_order,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
int swizzle_size
) {
result.bytes = problem_.bytes_with_problem_shape(operation_desc, problem_shape);
result.flops = problem_.flops_with_problem_shape(operation_desc, problem_shape);
set_argument(result, "m", problem_space, problem_shape.m());
set_argument(result, "n", problem_space, problem_shape.n());
set_argument(result, "k", problem_space, problem_shape.k());
set_argument(result, "raster_order", problem_space, library::to_string(raster_order));
set_argument(result, "swizzle_size", problem_space, swizzle_size);
set_argument(result, "cluster_m", problem_space, preferred_cluster[0]);
set_argument(result, "cluster_n", problem_space, preferred_cluster[1]);
set_argument(result, "cluster_k", problem_space, preferred_cluster[2]);
set_argument(result, "cluster_m_fallback", problem_space, fallback_cluster[0]);
set_argument(result, "cluster_n_fallback", problem_space, fallback_cluster[1]);
set_argument(result, "cluster_k_fallback", problem_space, fallback_cluster[2]);
}
/// Initializes the performance result
void GemmOperationProfiler::initialize_result_(
PerformanceResult &result,
@@ -723,14 +838,25 @@ Status GemmOperationProfiler::initialize_workspace(
bool allocate_device_tensors = options.execution_mode != ExecutionMode::kDryRun;
if (allocate_device_tensors) {
bool enable_deep_profiling = options.profiling.enable_kernel_performance_search;
int seed_shift = 0;
// When exhaustive performance search (deep profiling) option is enabled, device buffers are initialized to the largest problem shape
// so that later performance search can re-use those buffers.
int init_m = enable_deep_profiling ? std::max(int(problem_.m), problem_.problem_sizes.back().m()) : int(problem_.m);
int init_n = enable_deep_profiling ? std::max(int(problem_.n), problem_.problem_sizes.back().n()) : int(problem_.n);
int init_k = enable_deep_profiling ? std::max(int(problem_.k), problem_.problem_sizes.back().k()) : int(problem_.k);
int init_lda = enable_deep_profiling ? int(std::max(problem_.lda, problem_.leading_dims.back()[0])) : int(problem_.lda);
int init_ldb = enable_deep_profiling ? int(std::max(problem_.ldb, problem_.leading_dims.back()[1])) : int(problem_.ldb);
int init_ldc = enable_deep_profiling ? int(std::max(problem_.ldc, problem_.leading_dims.back()[2])) : int(problem_.ldc);
gemm_workspace_[i].A = device_context.allocate_and_initialize_tensor(
options,
"A",
operation_desc.A.element,
operation_desc.A.layout,
{int(problem_.m), int(problem_.k)},
{int(problem_.lda)},
{init_m, init_k},
{init_lda},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -741,8 +867,8 @@ Status GemmOperationProfiler::initialize_workspace(
"B",
operation_desc.B.element,
operation_desc.B.layout,
{int(problem_.k), int(problem_.n)},
{int(problem_.ldb)},
{init_k, init_n},
{init_ldb},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -753,8 +879,8 @@ Status GemmOperationProfiler::initialize_workspace(
"C",
operation_desc.C.element,
operation_desc.C.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
seed_shift++,
i // device_index
@@ -765,8 +891,8 @@ Status GemmOperationProfiler::initialize_workspace(
"D",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
@@ -776,13 +902,13 @@ Status GemmOperationProfiler::initialize_workspace(
"Reference",
operation_desc.D.element,
operation_desc.D.layout,
{int(problem_.m), int(problem_.n)},
{int(problem_.ldc)},
{init_m, init_n},
{init_ldc},
problem_.batch_count * gemm_workspace_[i].problem_count,
i // device_index
);
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
// Encoded tensor has the same shape and data type of the narrow data type tensor
@@ -830,14 +956,14 @@ Status GemmOperationProfiler::initialize_workspace(
i // device_index
);
}
}
} // End of "if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype)"
}
if (options.execution_mode != ExecutionMode::kDryRun) {
// NOTE: the leading non-batch strides are duplicated here for 3.0 API kernels
gemm_workspace_[i].arguments.problem_size = {int(problem_.m), int(problem_.n), int(problem_.k)};
gemm_workspace_[i].arguments.cluster_shape = {int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.cluster_shape_fallback = {int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)};
gemm_workspace_[i].arguments.split_k_slices = problem_.split_k_slices;
gemm_workspace_[i].arguments.batch_count = problem_.batch_count;
gemm_workspace_[i].arguments.lda = problem_.lda;
@@ -974,7 +1100,7 @@ bool GemmOperationProfiler::verify_cutlass(
gemm_workspace_[i].arguments.batch_stride_C = gemm_workspace_[i].C->batch_stride();
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
if (gemm_workspace_[i].arguments.is_mixed_dtype) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Scale and zero already generated in initialize_configuration(),
// A and B already generated in initialize_workspace(), signal
// GemmUniversal3xOperation::update_arguments_() (trigger by underlying_operation->run())
@@ -1299,7 +1425,7 @@ bool GemmOperationProfiler::verify_with_reference_(
cutlass::library::NumericTypeID element_A_for_reference = element_A;
cutlass::library::NumericTypeID element_B_for_reference = element_B;
if (gemm_workspace_[i].arguments.is_mixed_dtype && gemm_workspace_[i].arguments.dequantized_AB_ready[0]) {
if (gemm_workspace_[i].arguments.is_sm90_mixed_dtype) {
// Dequantized tensor has the same shape of the narrow data type tensor,
// and the same data type as the wide data type tensor
if (gemm_workspace_[i].arguments.wider_operand == cutlass::library::Sm90MixedInputWiderOperand::A) {
@@ -1444,42 +1570,213 @@ bool GemmOperationProfiler::profile(
if (options.profiling.provider_enabled(library::Provider::kCUTLASS)) {
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
// Initialize structure containing GEMM arguments
gemm_workspace_[i].arguments.A = gemm_workspace_[i].A->data();
gemm_workspace_[i].arguments.B = gemm_workspace_[i].B->data();
gemm_workspace_[i].arguments.C = gemm_workspace_[i].C->data();
gemm_workspace_[i].arguments.D = gemm_workspace_[i].Computed->data();
gemm_workspace_[i].arguments.alpha = problem_.alpha.data();
gemm_workspace_[i].arguments.beta = problem_.beta.data();
gemm_workspace_[i].arguments.pointer_mode = library::ScalarPointerMode::kHost;
gemm_workspace_[i].arguments.batch_stride_A = gemm_workspace_[i].A->batch_stride();
gemm_workspace_[i].arguments.batch_stride_B = gemm_workspace_[i].B->batch_stride();
gemm_workspace_[i].arguments.batch_stride_C = gemm_workspace_[i].C->batch_stride();
gemm_workspace_[i].arguments.batch_stride_D = gemm_workspace_[i].Computed->batch_stride();
// Case when we either screen the best performance number of kernels with or without a fixed problem shape fed in.
if (options.profiling.enable_kernel_performance_search || options.profiling.enable_best_kernel_for_fixed_shape) {
library::GemmDescription const &operation_desc =
static_cast<library::GemmDescription const &>(operation->description());
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
gemm_workspace_[i].arguments.D = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].arguments.alpha = problem_.alpha_one.data();
gemm_workspace_[i].arguments.beta = problem_.beta_zero.data();
auto min_cc = operation_desc.tile_description.minimum_compute_capability;
gemm_workspace_[i].reduction_arguments.workspace = gemm_workspace_[i].device_workspace.data();
gemm_workspace_[i].reduction_arguments.source = gemm_workspace_[i].C->data();
gemm_workspace_[i].reduction_arguments.destination = gemm_workspace_[i].Computed->data();
gemm_workspace_[i].reduction_arguments.alpha = problem_.alpha.data();
gemm_workspace_[i].reduction_arguments.beta = problem_.beta.data();
gemm_workspace_[i].reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
bool is_dynamic_cluster_enabled = (min_cc >= 100);
// Helper function wrapping up performance test with flexible parameters.
auto initialize_and_profile = [&](
PerformanceResult const &result,
gemm::GemmCoord const &problem_shape,
std::array<int64_t, 3> const &leading_dim,
std::array<int64_t, 3> const &preferred_cluster,
std::array<int64_t, 3> const &fallback_cluster,
cutlass::library::RasterOrder const &raster_order,
int swizzle_size) -> std::optional<PerformanceResult> {
for (size_t i = 0; i < gemm_workspace_.size(); ++i) {
// Initialize structure containing GEMM arguments
auto& workspace = gemm_workspace_[i];
workspace.arguments.A = workspace.A->data();
workspace.arguments.B = workspace.B->data();
workspace.arguments.C = workspace.C->data();
workspace.arguments.D = workspace.Computed->data();
workspace.arguments.alpha = problem_.alpha.data();
workspace.arguments.beta = problem_.beta.data();
workspace.arguments.pointer_mode = library::ScalarPointerMode::kHost;
workspace.arguments.batch_stride_A = workspace.A->batch_stride();
workspace.arguments.batch_stride_B = workspace.B->batch_stride();
workspace.arguments.batch_stride_C = workspace.C->batch_stride();
workspace.arguments.batch_stride_D = workspace.Computed->batch_stride();
if (problem_.split_k_mode == library::SplitKMode::kParallel) {
workspace.arguments.D = workspace.device_workspace.data();
workspace.arguments.alpha = problem_.alpha_one.data();
workspace.arguments.beta = problem_.beta_zero.data();
workspace.reduction_arguments.workspace = workspace.device_workspace.data();
workspace.reduction_arguments.source = workspace.C->data();
workspace.reduction_arguments.destination = workspace.Computed->data();
workspace.reduction_arguments.alpha = problem_.alpha.data();
workspace.reduction_arguments.beta = problem_.beta.data();
workspace.reduction_arguments.pointer_mode = library::ScalarPointerMode::kHost;
}
update_workspace_(workspace, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
const auto can_implement = operation->can_implement(&workspace.configuration, &workspace.arguments);
if (can_implement != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
library::Operation const* underlying_operation = operation;
cudaSetDevice(options.device.device_id(i));
uint64_t workspace_size = underlying_operation->get_host_workspace_size(&workspace.configuration);
workspace.host_workspace.resize(workspace_size, 0);
workspace_size = underlying_operation->get_device_workspace_size(&workspace.configuration,
&workspace.arguments);
bool is_sparse = operation_desc.tile_description.math_instruction.opcode_class == cutlass::library::OpcodeClassID::kSparseTensorOp;
if (is_sparse) {
// sparse gemm get_device_workspace_size() only return device workspace size per iteration
// Needs to multiply it w/ number of iteration
workspace_size *= workspace.problem_count;
}
workspace.device_workspace.reset(library::NumericTypeID::kU8, workspace_size);
Status status = Status::kSuccess;
if (is_sparse) {
uint8_t* profiler_workspaces[1];
profiler_workspaces[0] = reinterpret_cast<uint8_t*>(workspace.A->data());
// Sparse operations have a different initialize interface.
// initialize_with_profiler_workspace converts mxk tensorA to compressed mxk/sp tensorA and the tensorE
auto modifiable_underlying_op = const_cast<library::Operation*>(underlying_operation);
status = modifiable_underlying_op->initialize_with_profiler_workspace(
&workspace.configuration,
workspace.host_workspace.data(),
workspace.device_workspace.data(),
profiler_workspaces,
workspace.problem_count,
workspace.stream);
}
else {
status = underlying_operation->initialize(
&workspace.configuration,
workspace.host_workspace.data(),
workspace.device_workspace.data(),
workspace.stream);
}
if (status != Status::kSuccess) {
return std::nullopt; // Return nullopt to indicate failure
}
}
PerformanceResult curr_result(result);
update_result_(curr_result, operation_desc, problem_space, problem_shape, raster_order, preferred_cluster, fallback_cluster, swizzle_size);
curr_result.status = profile_cutlass_(
curr_result,
options,
operation,
nullptr,
nullptr,
nullptr
);
return curr_result;
};
// Helper function to test validity of fallback cluster shapes and preferred cluster shapes.
auto is_valid_dynamic_cluster_shape = [](const std::array<int64_t, 3>& preferred_cluster, const std::array<int64_t, 3>& fallback_cluster) {
for (size_t i = 0; i < 3; ++i) {
if (preferred_cluster[i] % fallback_cluster[i] != 0) {
return false;
}
}
return true;
};
// Helper function to select the best performance number among a list.
auto select_best_candidate = [&](std::vector<PerformanceResult> &candidates) {
assert(!candidates.empty() && "Candidates vector should not be empty");
auto best_iter = std::max_element(
candidates.begin(), candidates.end(),
[](PerformanceResult const &a, PerformanceResult const &b) {
return a.gflops_per_sec() < b.gflops_per_sec();
}
);
assert(best_iter != candidates.end() && "No candidate found despite non-empty candidates vector");
results_.push_back(std::move(*best_iter));
};
std::vector<PerformanceResult> candidates;
PerformanceResult result_base = results_.back();
results_.pop_back();
bool dynamic_cluster = int64_t(operation_desc.tile_description.cluster_shape.m()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.n()) == 0 ||
int64_t(operation_desc.tile_description.cluster_shape.k()) == 0;
std::vector<std::array<int64_t, 3>> preferred_clusters;
std::vector<std::array<int64_t, 3>> fallback_clusters;
// Only loop over built-in cluster shape lists for dynamic cluster kernels
// and for kernels that can leverage the dynamic cluster feature.
if (dynamic_cluster && is_dynamic_cluster_enabled) {
preferred_clusters = this->problem_.preferred_clusters;
fallback_clusters = this->problem_.fallback_clusters;
}
else {
preferred_clusters = {{int(problem_.cluster_m), int(problem_.cluster_n), int(problem_.cluster_k)}};
fallback_clusters = {{int(problem_.cluster_m_fallback), int(problem_.cluster_n_fallback), int(problem_.cluster_k_fallback)}};
}
for (auto preferred_cluster : preferred_clusters) {
for (auto fallback_cluster : fallback_clusters) {
if (dynamic_cluster && !is_valid_dynamic_cluster_shape(preferred_cluster, fallback_cluster)) {
continue;
}
for (auto swizzle_size : this->problem_.swizzle_sizes) {
for (auto raster_order : this->problem_.raster_orders) {
// With the fixed shape option turned on, only a specific problem shape is tested.
if (options.profiling.enable_best_kernel_for_fixed_shape) {
this->problem_.problem_sizes = {{int(this->problem_.m), int(this->problem_.n), int(this->problem_.k)}};
this->problem_.leading_dims = {{this->problem_.lda, this->problem_.ldb, this->problem_.ldc}};
}
for (int i = 0; i < int(this->problem_.problem_sizes.size()); i++) {
gemm::GemmCoord problem_shape = problem_.problem_sizes[i];
std::array<int64_t, 3> leading_dim = problem_.leading_dims[i];
auto result_opt = initialize_and_profile(result_base, problem_shape, leading_dim, preferred_cluster, fallback_cluster, raster_order, swizzle_size);
if (result_opt) { // Only add valid results
candidates.push_back(*result_opt);
}
}
}// for raster_order
}// for swizzle_size
}// for fallback_cluster
}// for swizzle_size
if (candidates.empty()) {
return false;
}
select_best_candidate(candidates);
}
// Basic case where we benchmark input parameters only.
else {
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
nullptr,
nullptr,
nullptr
);
}
results_.back().status = profile_cutlass_(
results_.back(),
options,
operation,
nullptr,
nullptr,
nullptr
);
}
return true;
}
@@ -1512,7 +1809,7 @@ Status GemmOperationProfiler::profile_cutlass_(
gemm_workspace_[dev_id].arguments.C = gemm_workspace_[dev_id].C->batch_data(problem_idx);
gemm_workspace_[dev_id].arguments.D = gemm_workspace_[dev_id].Computed->batch_data(problem_idx);
if (gemm_workspace_[dev_id].arguments.is_mixed_dtype) {
if (gemm_workspace_[dev_id].arguments.is_sm90_mixed_dtype) {
// Scale, zero, and dequantized tensors are already generated in
// verify_cutlass(), no need to re-generate them in profiling
gemm_workspace_[dev_id].arguments.generate_scale_and_zero = false;
tools/profiler/src/options.cu
+5 -1
View File
@@ -478,6 +478,8 @@ Options::Profiling::Profiling(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("profiling-duration", duration, 10);
cmdline.get_cmd_line_argument("min-iterations", min_iterations, 10);
cmdline.get_cmd_line_argument("use-cuda-graphs", use_cuda_graphs, false);
cmdline.get_cmd_line_argument("enable-kernel-performance-search", enable_kernel_performance_search, false);
cmdline.get_cmd_line_argument("enable-best-kernel-for-fixed-shape", enable_best_kernel_for_fixed_shape, false);
if (cmdline.check_cmd_line_flag("providers")) {
@@ -683,7 +685,9 @@ Options::Report::Report(cutlass::CommandLine const &cmdline) {
cmdline.get_cmd_line_argument("verbose", verbose, true);
cmdline.get_cmd_line_argument("sort-results", sort_results, false);
cmdline.get_cmd_line_argument("sort-results-flops-per-byte", sort_flops_per_byte, false);
cmdline.get_cmd_line_argument("sort-results-flops-per-sec", sort_flops_per_sec, false);
cmdline.get_cmd_line_argument("print-kernel-before-running", print_kernel_before_running, false);
}
tools/profiler/src/performance_report.cpp
+20 -3
View File
@@ -160,7 +160,7 @@ void PerformanceReport::append_result(PerformanceResult result) {
}
}
void PerformanceReport::sort_results(PerformanceResultVector &results) {
void PerformanceReport::sort_flops_per_byte(PerformanceResultVector &results) {
struct FlopsPerByteCompare
{
@@ -176,6 +176,19 @@ void PerformanceReport::sort_results(PerformanceResultVector &results) {
std::stable_sort(results.begin(), results.end(), FlopsPerByteCompare());
}
void PerformanceReport::sort_flops_per_sec(PerformanceResultVector &results) {
struct FlopsPerSecondCompare
{
bool operator()(const PerformanceResult &a, const PerformanceResult &b)
{
return a.gflops_per_sec() > b.gflops_per_sec();
}
};
std::stable_sort(results.begin(), results.end(), FlopsPerSecondCompare());
}
void PerformanceReport::append_results(PerformanceResultVector const &results) {
if (options_.report.verbose) {
@@ -195,8 +208,12 @@ PerformanceReport::~PerformanceReport() {
//
if (options_.report.verbose && !concatenated_results_.empty()) {
if (options_.report.sort_results) {
sort_results(concatenated_results_);
if (options_.report.sort_flops_per_byte) {
sort_flops_per_byte(concatenated_results_);
}
if (options_.report.sort_flops_per_sec) {
sort_flops_per_sec(concatenated_results_);
}
std::cout << "\n\n";
tools/util/include/cutlass/util/mixed_dtype_utils.hpp
+2 -2
View File
@@ -275,7 +275,7 @@ private:
};
// In the mainloop, PRMT selects 1 byte from only 8 bytes so the sign bit is handled in an extra PRMT.
// Here the encodings of positive values and negative values are unified (except for the sign bit).
// Here the encodings of positive values and negative values are unified (except for the sign bit).
// For instance, 1 becomes 0b0111, which is the same encoding as -1 (0b1111).
static bool unified_encode_int4b(cutlass::int4b_t const *block_in, cutlass::int4b_t *block_out, const size_t block_size) {
@@ -430,7 +430,7 @@ void reorder_tensor(
};
static_assert(has_major_mode(stride<0>(LayoutDst{})) ^ has_major_mode(stride<1>(LayoutDst{})),
"Could not find stride-1 mode in destination layout");
constexpr int N = shape_div(Int<8>{}, sizeof_bits<T>{});
constexpr int N = shape_div(Int<8>{}, Int<sizeof_bits_v<T>>{});
auto val_layout = conditional_return<has_major_mode(stride<0>(LayoutDst{}))>(
make_layout(make_shape(Int<N>{}, Int<1>{}), GenColMajor{}),
make_layout(make_shape(Int<1>{}, Int<N>{}), GenRowMajor{}));
tools/util/include/cutlass/util/reference/host/tensor_compare.h
+193
View File
@@ -55,6 +55,124 @@ namespace host {
namespace detail {
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
struct TensorGreatestErrorFunc {
//
// Data members
//
TensorView<Element, Layout> lhs;
TensorView<Element, Layout> rhs;
double result;
/// Ctor
TensorGreatestErrorFunc(
TensorView<Element, Layout> const &lhs_,
TensorView<Element, Layout> const &rhs_
) :
lhs(lhs_),
rhs(rhs_),
result(0.0) { }
/// Visits a coordinate
void operator()(Coord<Layout::kRank> const &coord) {
Element lhs_ = lhs.at(coord);
Element rhs_ = rhs.at(coord);
result = std::max(result, std::abs(double(lhs_) - double(rhs_)));
}
/// Returns true if equal
operator double() const {
return result;
}
};
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
struct TensorMREFunc {
//
// Data members
//
TensorView<Element, Layout> lhs;
TensorView<Element, Layout> rhs;
double sum;
uint64_t count;
static constexpr double epsilon = 1e-6;
/// Ctor
TensorMREFunc(
TensorView<Element, Layout> const &lhs_,
TensorView<Element, Layout> const &rhs_
) :
lhs(lhs_),
rhs(rhs_),
sum(0.0),
count(0) { }
/// Visits a coordinate
void operator()(Coord<Layout::kRank> const &coord) {
Element lhs_ = lhs.at(coord);
Element rhs_ = rhs.at(coord);
sum += std::abs(double(lhs_) - double(rhs_) / (double(rhs_) + epsilon));
++count;
}
/// Returns true if equal
operator double() const {
return sum / double(count);
}
};
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
struct TensorMSEFunc {
//
// Data members
//
TensorView<Element, Layout> lhs;
TensorView<Element, Layout> rhs;
double sum;
uint64_t count;
/// Ctor
TensorMSEFunc(
TensorView<Element, Layout> const &lhs_,
TensorView<Element, Layout> const &rhs_
) :
lhs(lhs_),
rhs(rhs_),
sum(0.0),
count(0) { }
/// Visits a coordinate
void operator()(Coord<Layout::kRank> const &coord) {
Element lhs_ = lhs.at(coord);
Element rhs_ = rhs.at(coord);
sum += std::pow((double(lhs_) - double(rhs_)), 2);
++count;
}
/// Returns true if equal
operator double() const {
return sum / double(count);
}
};
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
@@ -144,6 +262,81 @@ struct TensorRelativelyEqualsFunc {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns the Mean Squared Error between two tensors.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
double TensorMSE(
TensorView<Element, Layout> const &lhs,
TensorView<Element, Layout> const &rhs) {
// Extents must be identical
if (lhs.extent() != rhs.extent()) {
return -1;
}
detail::TensorMSEFunc<Element, Layout> func(lhs, rhs);
TensorForEach(
lhs.extent(),
func
);
return double(func);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns the Mean Relative Error between two tensors.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
double TensorMRE(
TensorView<Element, Layout> const &lhs,
TensorView<Element, Layout> const &rhs) {
// Extents must be identical
if (lhs.extent() != rhs.extent()) {
return -1;
}
detail::TensorMREFunc<Element, Layout> func(lhs, rhs);
TensorForEach(
lhs.extent(),
func
);
return double(func);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns the greatest error between two tensors.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
double TensorGreatestError(
TensorView<Element, Layout> const &lhs,
TensorView<Element, Layout> const &rhs) {
// Extents must be identical
if (lhs.extent() != rhs.extent()) {
return -1;
}
detail::TensorGreatestErrorFunc<Element, Layout> func(lhs, rhs);
TensorForEach(
lhs.extent(),
func
);
return double(func);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Returns true if two tensor views are equal.
template <
typename Element, ///< Element type