leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

v4.3 tag release update. (#2789)

Browse Source
This commit is contained in:
Junkai-Wu
2025-11-20 20:49:44 -05:00
committed by GitHub
parent 406e078b29
commit 8cd5bef43a
225 changed files with 23229 additions and 2813 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/library/src/grouped_gemm_operation_3x.hpp
+500
View File
@@ -869,5 +869,505 @@ public:
}
};
template <typename Operator_>
class MoeGroupedGemmOperation3xBase : public GemmOperation3xBase<Operator_> {
public:
using Operator = Operator_;
using OperatorArguments = typename Operator::Arguments;
using ElementA = typename Operator::ElementA;
using LayoutA = typename Operator::LayoutA;
using ElementB = typename Operator::ElementB;
using LayoutB = typename Operator::LayoutB;
using ElementC = typename Operator::ElementC;
using LayoutC = typename Operator::LayoutC;
using ElementD = typename Operator::ElementD;
using LayoutD = typename Operator::LayoutD;
using ElementAccumulator = typename Operator::ElementAccumulator;
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
using ArrayElementA = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementA;
using ArrayElementB = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementB;
using CollectiveMainloop = typename Operator::CollectiveMainloop;
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
using StrideC = typename Operator::GemmKernel::StrideC;
using StrideD = typename Operator::GemmKernel::StrideD;
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementA>();
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementB>();
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
MoeGroupedGemmOperation3xBase(char const* name = "unknown_gemm")
: GemmOperation3xBase<Operator_>(name, GemmKind::kGrouped) {
this->description_.is_moe = true;
this->description_.kind = OperationKind::kGroupedGemm;
this->description_.name = name;
this->description_.provider = Provider::kCUTLASS;
this->description_.gemm = GemmOperation3xBase<Operator_>::description_;
this->description_.tile_description = this->description_.gemm.tile_description;
};
public:
// mutable CudaBuffer strideC_device;
// mutable CudaBuffer strideD_device;
/// Returns the description of the GEMM operation
virtual OperationDescription const& description() const override final { return description_; }
/// Gets the host-side workspace
uint64_t get_host_workspace_size(void const* configuration) const override final {
return sizeof(Operator);
}
protected:
library::GroupedGemmDescription description_;
/// Constructs the arguments structure given the configuration and arguments
Status update_arguments_base(
OperatorArguments& operator_args,
GemmGroupedArguments const& arguments) const {
operator_args.mode = cutlass::gemm::GemmUniversalMode::kGrouped;
int M= arguments.max_problem_size_3x[0];
int N = arguments.max_problem_size_3x[1];
int K = arguments.max_problem_size_3x[2];
int L = arguments.problem_count;
operator_args.problem_shape = {
M,
N,
K,
L,
arguments.tokens_per_expert,
arguments.tokens_per_expert_host
};
if constexpr (IsRuntimeDataType) {
using RuntimeDataTypeA = typename Operator::GemmKernel::CollectiveMainloop::RuntimeDataTypeA;
using RuntimeDataTypeB = typename Operator::GemmKernel::CollectiveMainloop::RuntimeDataTypeB;
static_assert(cute::is_same_v<RuntimeDataTypeA, RuntimeDataTypeB>,
"RuntimeDataTypeA/B should be identical, either MXF8F6F4Format or MXF4Format");
using RuntimeDatatypeArg = RuntimeDataTypeA;
auto mapping = [](RuntimeDatatype type) {
if constexpr (cute::is_same_v<RuntimeDatatypeArg, cute::UMMA::MXF8F6F4Format>) {
if (type == RuntimeDatatype::kE5M2) {
return cute::UMMA::MXF8F6F4Format::E5M2;
}
else if (type == RuntimeDatatype::kE4M3) {
return cute::UMMA::MXF8F6F4Format::E4M3;
}
else if (type == RuntimeDatatype::kE3M2) {
return cute::UMMA::MXF8F6F4Format::E3M2;
}
else if (type == RuntimeDatatype::kE2M3) {
return cute::UMMA::MXF8F6F4Format::E2M3;
}
else if (type == RuntimeDatatype::kE2M1) {
return cute::UMMA::MXF8F6F4Format::E2M1;
}
else {
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && CUTLASS_DEBUG_TRACE_LEVEL >= 1
std::cerr << "Invalid input datatype specified. Running with e4m3." << std::endl;
#endif
return cute::UMMA::MXF8F6F4Format::E4M3;
}
}
else if constexpr (cute::is_same_v<RuntimeDatatypeArg, cute::UMMA::MXF4Format>) {
if (type == RuntimeDatatype::kE2M1) {
return cute::UMMA::MXF4Format::E2M1;
}
else {
#if defined(CUTLASS_DEBUG_TRACE_LEVEL) && CUTLASS_DEBUG_TRACE_LEVEL >= 1
std::cerr << "Invalid input datatype specified. Running with e2m1." << std::endl;
#endif
return cute::UMMA::MXF4Format::E2M1;
}
}
// BlockScaled kernels receive either MXF4Format or MXF8F6F4Format runtime datatype
CUTE_GCC_UNREACHABLE;
};
operator_args.mainloop.runtime_data_type_a = mapping(arguments.runtime_input_datatype_a);
operator_args.mainloop.runtime_data_type_b = mapping(arguments.runtime_input_datatype_b);
}
operator_args.epilogue.ptr_C = static_cast<ElementC const*>(arguments.ptr_C);
operator_args.epilogue.ptr_D = static_cast<ElementD*>(arguments.ptr_D);
operator_args.epilogue.dC = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
operator_args.epilogue.dD = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
/* Query device SM count and max active clusters to pass onto the kernel as an argument, where needed */
operator_args.hw_info.sm_count = arguments.sm_count;
if constexpr (Operator::ArchTag::kMinComputeCapability >= 90) {
operator_args.hw_info.max_active_clusters = arguments.max_active_clusters;
}
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.max_swizzle_size)>) {
operator_args.scheduler.max_swizzle_size = arguments.swizzle_size;
}
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.raster_order)>) {
using Enum_t = decltype(operator_args.scheduler.raster_order);
switch (arguments.raster_order) {
case RasterOrder::kAlongN:
operator_args.scheduler.raster_order = Enum_t::AlongN;
break;
case RasterOrder::kAlongM:
operator_args.scheduler.raster_order = Enum_t::AlongM;
break;
default:
operator_args.scheduler.raster_order = Enum_t::Heuristic;
}
}
if constexpr (Operator::ArchTag::kMinComputeCapability >= 100) {
operator_args.hw_info.cluster_shape =
dim3(arguments.cluster_shape.m(), arguments.cluster_shape.n(), arguments.cluster_shape.k());
operator_args.hw_info.cluster_shape_fallback = dim3(
arguments.cluster_shape_fallback.m(),
arguments.cluster_shape_fallback.n(),
arguments.cluster_shape_fallback.k());
}
return Status::kSuccess;
}
template <typename FusionArgs>
static Status update_fusion_args(FusionArgs& fusion_args, GemmGroupedArguments const& arguments) {
if (arguments.pointer_mode == ScalarPointerMode::kHost) {
fusion_args.alpha = *static_cast<ElementCompute const*>(arguments.alpha);
fusion_args.beta = *static_cast<ElementCompute const*>(arguments.beta);
fusion_args.alpha_ptr = nullptr;
fusion_args.beta_ptr = nullptr;
return Status::kSuccess;
}
else if (arguments.pointer_mode == ScalarPointerMode::kDevice) {
fusion_args.alpha = 0;
fusion_args.beta = 0;
fusion_args.alpha_ptr = static_cast<ElementCompute const*>(arguments.alpha);
fusion_args.beta_ptr = static_cast<ElementCompute const*>(arguments.beta);
return Status::kSuccess;
}
else {
return Status::kErrorInvalidProblem;
}
}
};
template<typename Operator_>
class MoeGroupedGemmUniversal3xOperation : public MoeGroupedGemmOperation3xBase<Operator_> {
public:
using Base = MoeGroupedGemmOperation3xBase<Operator_>;
using Operator = Operator_;
using OperatorArguments = typename Operator::Arguments;
MoeGroupedGemmUniversal3xOperation(char const* name = "unknown_gemm")
: MoeGroupedGemmOperation3xBase<Operator_>(name) {
}
~MoeGroupedGemmUniversal3xOperation() override = default;
protected:
template <class FusionArgs, class = void> struct UpdateFusionArgs {
static Status update_(FusionArgs const& fusion_args, GemmGroupedArguments const& arguments) {
// If a custom EVT is instantiated then it is the users's responsibility
// to ensure alpha and beta are updated appropriately
return Status::kSuccess;
}
};
template <class FusionArgs>
struct UpdateFusionArgs<FusionArgs, cute::void_t<decltype(FusionArgs{}.alpha)>> {
static Status update_(FusionArgs& fusion_args, GemmGroupedArguments const& arguments) {
return MoeGroupedGemmOperation3xBase<Operator>::update_fusion_args(fusion_args, arguments);
}
};
/// Constructs the arguments structure given the configuration and arguments
Status
update_arguments_(OperatorArguments& operator_args, GemmGroupedArguments const* arguments) const {
Status status = UpdateFusionArgs<decltype(operator_args.epilogue.thread)>::update_(
operator_args.epilogue.thread,
*arguments);
if (status != Status::kSuccess) {
return status;
}
status = this->update_arguments_base(operator_args, *arguments);
operator_args.mainloop.ptr_A = static_cast<typename Base::ArrayElementA const*>(arguments->ptr_A);
operator_args.mainloop.ptr_B = static_cast<typename Base::ArrayElementB const*>(arguments->ptr_B);
return status;
}
public:
/// Returns success if the operation can proceed
Status can_implement([[maybe_unused]] void const* configuration_ptr, void const* arguments_ptr)
const override {
GemmGroupedArguments const* arguments = static_cast<GemmGroupedArguments const*>(arguments_ptr);
OperatorArguments args;
auto status = update_arguments_(args, arguments);
if (status != Status::kSuccess) {
return status;
}
status = Operator::can_implement(args);
return status;
}
/// Gets the device-side workspace
uint64_t get_device_workspace_size(void const* configuration_ptr, void const* arguments_ptr)
const override {
OperatorArguments args;
auto status = update_arguments_(args, static_cast<GemmGroupedArguments const*>(arguments_ptr));
if (status != Status::kSuccess) {
return 0;
}
uint64_t size = Operator::get_workspace_size(args);
return size;
}
/// Initializes the workspace
/// **** CAUTION ****
/// Must be called when ldc, or ldd change.
/// The CUTLASS library stores the operations in a type-
/// erased manifest. Therefore, only this class knows
/// the type of strideC, and strideD.
/// Since grouped GEMM needs to allocate storage for
/// the strides on device, the concrete type of the stride
/// must be known in order to copy in the correct memory
/// layout on device.
Status initialize(
void const* configuration_ptr,
void* host_workspace,
void* device_workspace,
cudaStream_t stream = nullptr) const override {
Operator* op = new (host_workspace) Operator;
return Status::kSuccess;
}
/// **** CAUTION ****
/// initialize() must be called if lda, ldb, ldc, or ldd change.
Status run(
void const* arguments_ptr,
void* host_workspace,
void* device_workspace = nullptr,
cudaStream_t stream = nullptr) const override {
OperatorArguments operator_args;
auto const& args = *static_cast<GemmGroupedArguments const*>(arguments_ptr);
Status status = update_arguments_(operator_args, &args);
if (status != Status::kSuccess) {
return status;
}
Operator* op = static_cast<Operator*>(host_workspace);
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
status = op->run(operator_args, device_workspace, stream, nullptr, args.use_pdl);
return status;
}
// Set arguments that should only be set once before verifying or profiling the kernel.
// This should encompass any expensive operations that don't vary from run to run
// (e.g., max_active_clusters).
Status initialize_with_arguments(void* arguments_ptr) const override {
if constexpr (Operator::ArchTag::kMinComputeCapability < 90) {
return Status::kSuccess;
}
GemmGroupedArguments* args = static_cast<GemmGroupedArguments*>(arguments_ptr);
dim3 cluster_dims;
if constexpr (cute::is_static_v<typename Operator::GemmKernel::ClusterShape>) {
cluster_dims = dim3(
cute::size<0>(typename Operator::GemmKernel::ClusterShape{}),
cute::size<1>(typename Operator::GemmKernel::ClusterShape{}),
cute::size<2>(typename Operator::GemmKernel::ClusterShape{})
);
}
else {
cluster_dims = dim3(
args->cluster_shape.m(),
args->cluster_shape.n(),
args->cluster_shape.k()
);
}
uint32_t threads_per_block = Operator::GemmKernel::MaxThreadsPerBlock;
void const* kernel_ptr = (void*)(device_kernel<typename Operator::GemmKernel>);
args->max_active_clusters = cutlass::KernelHardwareInfo::query_device_max_active_clusters(
cluster_dims,
threads_per_block,
kernel_ptr);
if (args->max_active_clusters == 0) {
std::cerr << "Max Active Clusters could not be queried. "
<< "Falling back to heuristics mode (static cluster shape) or preferred cluster mode.\n";
}
return Status::kSuccess;
}
};
template<typename Operator_>
class BlockScaledMoeGroupedGemmUniversal3xOperation : public MoeGroupedGemmOperation3xBase<Operator_> {
public:
using Base = MoeGroupedGemmOperation3xBase<Operator_>;
using Operator = Operator_;
using OperatorArguments = typename Operator::Arguments;
using ElementD = typename Operator::ElementD;
using LayoutD = typename Operator::LayoutD;
using ElementAccumulator = typename Operator::ElementAccumulator;
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
using CollectiveMainloop = typename Operator::CollectiveMainloop;
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
using ElementSF = typename Operator::CollectiveMainloop::ElementSF;
using TiledMma = typename Operator::CollectiveMainloop::TiledMma;
constexpr static int SFVecSize = TiledMma::SFVecSize;
static constexpr bool epilogue_scalefactor_generation = not cute::is_same_v<typename ThreadEpilogueOp::ElementBlockScaleFactor, void>;
static constexpr int32_t SFD_VectorSize = epilogue_scalefactor_generation ? ThreadEpilogueOp::SFVecSize : SFVecSize;
using ElementSFD = cute::conditional_t<epilogue_scalefactor_generation, typename ThreadEpilogueOp::ElementBlockScaleFactor, void>;
using LayoutSFD = cute::conditional_t<epilogue_scalefactor_generation, typename ThreadEpilogueOp::GmemLayoutTagScalefactor, LayoutD>;
BlockScaledMoeGroupedGemmUniversal3xOperation(char const* name = "unknown_gemm")
: MoeGroupedGemmOperation3xBase<Operator_>(name) {
BlockScaleDescription block_scaled_desc{};
block_scaled_desc.kind = OperationKind::kBlockScaledGemm;
block_scaled_desc.SFA.element = NumericTypeMap<ElementSF>::kId;
block_scaled_desc.SFA.layout = LayoutTypeID::kRowMajor;
block_scaled_desc.SFA.alignment = 128;
block_scaled_desc.SFA.log_extent_range = 32;
block_scaled_desc.SFA.log_stride_range = 32;
block_scaled_desc.SFB.element = NumericTypeMap<ElementSF>::kId;
block_scaled_desc.SFB.layout = LayoutTypeID::kRowMajor;
block_scaled_desc.SFB.alignment = 128;
block_scaled_desc.SFB.log_extent_range = 32;
block_scaled_desc.SFB.log_stride_range = 32;
block_scaled_desc.SFMVecSize = 1;
block_scaled_desc.SFNVecSize = 1;
block_scaled_desc.SFKVecSize = SFVecSize;
block_scaled_desc.SFD = make_TensorDescription<ElementSFD, LayoutSFD>(128);
block_scaled_desc.EpilogueSFVecSize = SFD_VectorSize;
this->description_.block_scales = block_scaled_desc;
}
~BlockScaledMoeGroupedGemmUniversal3xOperation() override = default;
protected:
template <class FusionArgs, class = void> struct UpdateFusionArgs {
static Status update_(FusionArgs const& fusion_args, GroupedGemmBlockScaledArguments const& arguments) {
// If a custom EVT is instantiated then it is the users's responsibility
// to ensure alpha and beta are updated appropriately
return Status::kSuccess;
}
};
template <class FusionArgs>
struct UpdateFusionArgs<FusionArgs, cute::void_t<decltype(FusionArgs{}.alpha)>> {
static Status
update_(FusionArgs& fusion_args, GroupedGemmBlockScaledArguments const& arguments) {
if constexpr (epilogue_scalefactor_generation) {
fusion_args.block_scale_factor_ptr = static_cast<ElementSFD*>(arguments.SFD);
fusion_args.norm_constant_ptr = static_cast<ElementCompute const*>(arguments.norm_constant);
}
return MoeGroupedGemmOperation3xBase<Operator>::update_fusion_args(fusion_args, arguments);
}
};
public:
/// Returns success if the operation can proceed
Status can_implement([[maybe_unused]] void const* configuration_ptr, void const* arguments_ptr)
const override {
GroupedGemmBlockScaledArguments const* arguments =
static_cast<GroupedGemmBlockScaledArguments const*>(arguments_ptr);
OperatorArguments args;
auto status = update_arguments_(args, arguments);
if (status != Status::kSuccess) {
return status;
}
status = Operator::can_implement(args);
return status;
}
Status update_arguments_(
OperatorArguments& operator_args,
GroupedGemmBlockScaledArguments const* arguments) const {
Status status = UpdateFusionArgs<decltype(operator_args.epilogue.thread)>::update_(
operator_args.epilogue.thread,
*arguments);
if (status != Status::kSuccess) {
return status;
}
operator_args.mainloop.ptr_A = static_cast<typename Base::ArrayElementA const*>(arguments->ptr_A);
operator_args.mainloop.ptr_B = static_cast<typename Base::ArrayElementB const*>(arguments->ptr_B);
operator_args.mainloop.ptr_SFA = static_cast<const ElementSF*>(arguments->SFA);
operator_args.mainloop.ptr_SFB = static_cast<const ElementSF*>(arguments->SFB);
return this->update_arguments_base(operator_args, *arguments);
}
uint64_t get_device_workspace_size(void const* configuration_ptr, void const* arguments_ptr)
const override {
OperatorArguments args;
auto status =
update_arguments_(args, static_cast<GroupedGemmBlockScaledArguments const*>(arguments_ptr));
if (status != Status::kSuccess) {
return 0;
}
uint64_t size = Operator::get_workspace_size(args);
return size;
}
/// Initializes the workspace
/// **** CAUTION ****
/// Must be called when ldc, or ldd change.
/// The CUTLASS library stores the operations in a type-
/// erased manifest. Therefore, only this class knows
/// the type of strideA, strideB, strideC, and strideD.
/// Since grouped GEMM needs to allocate storage for
/// the strides on device, the concrete type of the stride
/// must be known in order to copy in the correct memory
/// layout on device.
Status initialize(
void const* configuration_ptr,
void* host_workspace,
void* device_workspace,
cudaStream_t stream = nullptr) const override {
Operator* op = new (host_workspace) Operator;
return Status::kSuccess;
}
/// **** CAUTION ****
/// initialize() must be called if ldc, or ldd change.
Status run(
void const* arguments_ptr,
void* host_workspace,
void* device_workspace = nullptr,
cudaStream_t stream = nullptr) const override {
OperatorArguments operator_args;
auto const& args = *static_cast<GroupedGemmBlockScaledArguments const*>(arguments_ptr);
Status status = update_arguments_(operator_args, &args);
if (status != Status::kSuccess) {
return status;
}
Operator* op = static_cast<Operator*>(host_workspace);
status = op->run(operator_args, device_workspace, stream, nullptr);
return status;
}
};
} // namespace cutlass::library
tools/library/src/handle.cu
+3
View File
@@ -279,6 +279,9 @@ static int gemm_problem_alignment(
int max_element_alignment = 0;
for (NumericTypeID type_id : elements) {
if (library::sizeof_bits(type_id)==0) {
continue;
}
int element_alignment = max_alignment_in_bytes * 8 / library::sizeof_bits(type_id);
max_element_alignment = std::max(max_element_alignment, element_alignment);
}
tools/library/src/reference/block_scaled_gemm_fp4a_vs16.cu
+5
View File
@@ -74,6 +74,11 @@ void initialize_block_scaled_gemm_reference_operations_fp4a_vs16(Manifest &manif
16 /*EpilogueSFVecSize*/
>(manifest);
make_block_scaled_gemm_tn<
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 16 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm_tn<
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
tools/library/src/reference/block_scaled_gemm_fp4a_vs32.cu
+9
View File
@@ -73,6 +73,15 @@ void initialize_block_scaled_gemm_reference_operations_fp4a_vs32(Manifest &manif
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
bfloat16_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 32 /*SFVecSize*/
>(manifest);
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, bfloat16_t /*D*/, 32 /*SFVecSize*/
>(manifest);
// With SF generation reference
make_block_scaled_gemm<
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
tools/library/src/reference/gemm_e4m3a_e4m3out.cu
+18
View File
@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e4m3a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e4m3a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
tools/library/src/reference/gemm_e4m3a_e5m2out.cu
+18
View File
@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e4m3a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e4m3a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
tools/library/src/reference/gemm_e5m2a_e4m3out.cu
+18
View File
@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e5m2a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
@@ -65,6 +74,15 @@ void initialize_gemm_reference_operations_e5m2a_e4m3out(Manifest &manifest) {
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
tools/library/src/reference/gemm_e5m2a_e5m2out.cu
+9
View File
@@ -56,6 +56,15 @@ void initialize_gemm_reference_operations_e5m2a_e5m2out(Manifest &manifest) {
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
tools/library/src/reference/gemm_f4_f4_f32.cu
+39
View File
@@ -97,7 +97,46 @@ void initialize_gemm_reference_operations_f4_f4_f32(Manifest &manifest) {
float, // ElementAccumulator
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f4_f6_f32.cu
+39
View File
@@ -99,6 +99,45 @@ void initialize_gemm_reference_operations_f4_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f4_f8_f32.cu
+79
View File
@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
@@ -139,6 +179,45 @@ void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e2m1_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f6_f4_f32.cu
+41
View File
@@ -99,6 +99,47 @@ void initialize_gemm_reference_operations_f6_f4_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f6_f6_f32.cu
+40
View File
@@ -98,6 +98,46 @@ void initialize_gemm_reference_operations_f6_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f6_f8_f32.cu
+39
View File
@@ -99,6 +99,45 @@ void initialize_gemm_reference_operations_f6_f8_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e3m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f8_f4_f32.cu
+40
View File
@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f8_f4_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e2m1_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_f8_f6_f32.cu
+40
View File
@@ -99,6 +99,46 @@ void initialize_gemm_reference_operations_f8_f6_f32(Manifest &manifest) {
float // ElementD
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e4m3_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float_e5m2_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e3m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp32out.cu
+61
View File
@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float // ElementAccumulator
>(manifest);
make_gemm_real_canonical_layouts<
float, // ElementA
float, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
@@ -62,6 +71,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
@@ -69,6 +87,14 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
tfloat32_t,
tfloat32_t,
void,
float,
float,
tfloat32_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
@@ -78,6 +104,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
@@ -86,6 +121,14 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
@@ -94,6 +137,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
@@ -101,6 +153,15 @@ void initialize_gemm_reference_operations_fp32out(Manifest &manifest) {
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp8in_bf16out.cu
+36
View File
@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_bf16out(Manifest &manifest) {
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
bfloat16_t // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp8in_fp16out.cu
+36
View File
@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_fp16out(Manifest &manifest) {
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
half_t // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp8in_fp32out.cu
+36
View File
@@ -82,6 +82,42 @@ void initialize_gemm_reference_operations_fp8in_fp32out(Manifest &manifest) {
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e4m3_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e4m3_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
make_gemm_real_canonical_layouts<
float_e5m2_t, // ElementA
float_e5m2_t, // ElementB
void, // ElementC
float, // ElementScalar
float, // ElementAccumulator
float // ElementD
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp_mixed_input.cu
+183
View File
@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -61,6 +70,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
@@ -68,6 +86,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -75,6 +103,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
@@ -82,6 +119,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
@@ -89,6 +135,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -96,6 +151,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -103,6 +167,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -110,6 +183,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -117,6 +200,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
@@ -124,6 +216,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
int8_t,
void,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
@@ -131,6 +232,16 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
uint8_t,
void,
float,
float,
half_t
>(manifest);
// bfloat16_t mixed with 8-bit integer input
make_gemm_real_canonical_layouts<
int8_t,
@@ -139,6 +250,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
@@ -146,6 +266,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
@@ -153,6 +282,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
@@ -160,6 +298,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
uint8_t,
bfloat16_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
@@ -167,6 +314,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
@@ -174,6 +330,15 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
void,
float,
float,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
@@ -181,12 +346,30 @@ void initialize_gemm_reference_operations_fp_mixed_input(Manifest &manifest) {
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
int8_t,
void,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
bfloat16_t,
float
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
uint8_t,
void,
float,
float,
bfloat16_t
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_fp_other.cu
+54
View File
@@ -54,6 +54,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
void,
half_t,
half_t,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
@@ -62,6 +71,24 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
half_t
>(manifest);
make_gemm_real_canonical_layouts<
half_t,
half_t,
float,
float,
float,
half_t
>(manifest);
make_gemm_real_canonical_layouts<
bfloat16_t,
bfloat16_t,
float,
float,
float,
bfloat16_t
>(manifest);
make_gemm_real_canonical_layouts<
double,
double,
@@ -70,6 +97,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
double
>(manifest);
make_gemm_real_canonical_layouts<
double,
double,
void,
double,
double,
double
>(manifest);
make_gemm_complex_canonical_layouts<
complex<float>,
complex<float>,
@@ -78,6 +114,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
complex<float>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<float>,
complex<float>,
void,
complex<float>,
complex<float>,
complex<float>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<double>,
complex<double>,
@@ -85,6 +130,15 @@ void initialize_gemm_reference_operations_fp_other(Manifest &manifest) {
complex<double>,
complex<double>
>(manifest);
make_gemm_complex_canonical_layouts<
complex<double>,
complex<double>,
void,
complex<double>,
complex<double>,
complex<double>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_int4.cu
+75
View File
@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
64,
int4b_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -90,7 +122,28 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
64,
uint4b_t,
uint4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +161,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
uint4b_t,
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -118,6 +182,17 @@ void initialize_gemm_reference_operations_int4(Manifest &manifest) {
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_int8_interleaved_32.cu
+75
View File
@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
32,
int8_t,
int8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
int8_t,
int8_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
@@ -90,7 +122,28 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
32,
uint8_t,
uint8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +161,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
NumericConverterClamp<uint8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
uint8_t,
NumericConverterClamp<uint8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
@@ -118,6 +182,17 @@ void initialize_gemm_reference_operations_int8_interleaved_32(Manifest &manifest
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_interleaved_layouts<
32,
uint8_t,
uint8_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_int8_interleaved_64.cu
+76
View File
@@ -59,7 +59,28 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
64,
int4b_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -77,6 +98,18 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
int4b_t,
int4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -90,7 +123,28 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
64,
uint4b_t,
uint4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int32_t,
@@ -108,6 +162,17 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
uint4b_t,
NumericConverterClamp<uint4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
@@ -118,6 +183,17 @@ void initialize_gemm_reference_operations_int8_interleaved_64(Manifest &manifest
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
make_gemm_interleaved_layouts<
64,
uint4b_t,
uint4b_t,
void,
float,
int32_t,
int4b_t,
NumericConverterClamp<int4b_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_int_mixed_input.cu
+78
View File
@@ -57,7 +57,26 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
int8_t,
int32_t,
int32_t,
int8_t,
NumericConverterClamp<int8_t, int32_t>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
int32_t,
int32_t,
int8_t,
@@ -77,7 +96,27 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
int8_t,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int4b_t,
int8_t,
void,
float,
int32_t,
int8_t,
@@ -91,6 +130,15 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
int32_t,
int32_t,
int32_t
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
@@ -104,7 +152,27 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
int32_t,
int32_t,
int8_t,
NumericConverterClamp<int8_t, int32_t>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
int32_t,
float,
int32_t,
int32_t,
NumericConverterClamp<int32_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
float,
int32_t,
int32_t,
@@ -120,6 +188,16 @@ void initialize_gemm_reference_operations_int_mixed_input(Manifest &manifest) {
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
make_gemm_real_canonical_layouts<
int8_t,
int4b_t,
void,
float,
int32_t,
int8_t,
NumericConverterClamp<int8_t, float>
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_reference_operation.h
+6 -5
View File
@@ -86,7 +86,8 @@ public:
using ElementC = ElementC_;
using LayoutC = LayoutC_;
using ElementD = ElementD_;
using TensorRefC = TensorRef<ElementC, LayoutC>;
using NonVoidElementC = cute::conditional_t<cute::is_void_v<ElementC>, ElementD, ElementC>;
using TensorRefC = TensorRef<NonVoidElementC, LayoutC>;
using TensorRefD = TensorRef<ElementD, LayoutC>;
using ElementCompute = ElementCompute_;
using ElementAccumulator = ElementAccumulator_;
@@ -199,7 +200,7 @@ public:
TensorRefA ref_A{static_cast<ElementA *>(const_cast<void *>(args.A)), LayoutA(int(config.lda))};
TensorRefB ref_B{static_cast<ElementB *>(const_cast<void *>(args.B)), LayoutB(int(config.ldb))};
TensorRefC ref_C{static_cast<ElementC *>(const_cast<void *>(args.C)), LayoutC(int(config.ldc))};
TensorRefC ref_C{static_cast<NonVoidElementC *>(const_cast<void *>(args.C)), LayoutC(int(config.ldc))};
TensorRefD ref_D{static_cast<ElementD *>(args.D), LayoutC(int(config.ldd))};
if (kProvider == Provider::kReferenceHost) {
@@ -209,7 +210,7 @@ public:
LayoutA,
ElementB,
LayoutB,
ElementC,
NonVoidElementC,
LayoutC,
ElementCompute,
ElementAccumulator,
@@ -243,7 +244,7 @@ public:
LayoutA,
ElementB,
LayoutB,
ElementC,
NonVoidElementC,
LayoutC,
ElementCompute,
ElementAccumulator,
@@ -434,7 +435,7 @@ template <
typename ElementCompute_,
typename ElementAccumulator_ = ElementCompute_,
typename ElementD_ = ElementC_,
typename ConvertOp_ = NumericConverter<ElementC_, ElementCompute_>,
typename ConvertOp_ = NumericConverter<ElementD_, ElementCompute_>,
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
>
void make_gemm_interleaved_layouts(Manifest &manifest) {
tools/library/src/reference/gemm_s8_s8_s32.cu
+74
View File
@@ -135,6 +135,80 @@ void initialize_gemm_reference_operations_s8_s8_s32(Manifest &manifest) {
half_t, // ElementD
NumericConverterClamp<half_t, float> // From Scalar to D
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, int32_t> // From Scalar to D
>(manifest);
// 5.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 6.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
float // ElementD
>(manifest);
// 7.
make_gemm_real_canonical_layouts<
int8_t, // ElementA
int8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
half_t, // ElementD
NumericConverterClamp<half_t, float> // From Scalar to D
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/library/src/reference/gemm_u8_u8_s32.cu
+43
View File
@@ -98,6 +98,49 @@ void initialize_gemm_reference_operations_u8_u8_s32(Manifest &manifest) {
NumericConverterClamp<uint8_t, float> // From Scalar to D
>(manifest);
// 1.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
int32_t, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t // ElementD
>(manifest);
// 2.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int32_t, // ElementD
NumericConverterClamp<int32_t, float> // From Scalar to D
>(manifest);
// 3.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
int8_t, // ElementD
NumericConverterClamp<int8_t, float> // From Scalar to D
>(manifest);
// 4.
make_gemm_real_canonical_layouts<
uint8_t, // ElementA
uint8_t, // ElementB
void, // ElementC
float, // ElementScalar / ElementCompute
int32_t, // ElementAccumulator
uint8_t, // ElementD
NumericConverterClamp<uint8_t, float> // From Scalar to D
>(manifest);
}
///////////////////////////////////////////////////////////////////////////////////////////////////