CUTLASS 3.8 Release (#2059)
* CUTLASS 3.8 Release * update * Update README.md * Revert "Update README.md" This reverts commit b353e36fe83e0815f99b44e46c0c95494c44726b. * update * update --------- Co-authored-by: Haicheng Wu <57973641+hwu36@users.noreply.github.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
mihir-awatramani
co-authored by
Haicheng Wu
Haicheng Wu
Haicheng Wu
Haicheng Wu
parent
9eb01fa0b0
commit
389e493055
@@ -0,0 +1,450 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines operations for all GEMM operation kinds in CUTLASS Library.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/detail/collective.hpp"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "library_internal.h"
|
||||
#include "gemm_operation_3x.hpp"
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass::library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Operator_>
|
||||
class BlockScaledGemmUniversal3xOperation : public GemmOperation3xBase<Operator_> {
|
||||
public:
|
||||
using Operator = Operator_;
|
||||
using OperatorArguments = typename Operator::Arguments;
|
||||
using ElementA = typename Operator::CollectiveMainloop::ElementA;
|
||||
using ElementSFA = typename Operator::CollectiveMainloop::ElementSF;
|
||||
using LayoutA = typename Operator::LayoutA;
|
||||
using ElementB = typename Operator::CollectiveMainloop::ElementB;
|
||||
using ElementSFB = typename Operator::CollectiveMainloop::ElementSF;
|
||||
using LayoutB = typename Operator::LayoutB;
|
||||
using ElementC = typename Operator::ElementC;
|
||||
using LayoutC = typename Operator::LayoutC;
|
||||
using ElementD = typename Operator::ElementD;
|
||||
using LayoutD = typename Operator::LayoutD;
|
||||
using ElementAccumulator = typename Operator::ElementAccumulator;
|
||||
using ElementCompute = typename Operator::EpilogueOutputOp::ElementCompute;
|
||||
|
||||
using TiledMma = typename Operator::CollectiveMainloop::TiledMma;
|
||||
constexpr static int SFVecSize = TiledMma::SFVecSize;
|
||||
|
||||
using CollectiveMainloop = typename Operator::CollectiveMainloop;
|
||||
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
|
||||
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
|
||||
|
||||
using Sm100BlkScaledConfig = typename CollectiveMainloop::Sm100BlkScaledConfig;
|
||||
|
||||
static constexpr bool epilogue_scalefactor_generation = not cute::is_same_v<typename ThreadEpilogueOp::ElementBlockScaleFactor, void>;
|
||||
static constexpr int32_t SFD_VectorSize = epilogue_scalefactor_generation ? ThreadEpilogueOp::SFVecSize : SFVecSize;
|
||||
using ElementSFD = cute::conditional_t<epilogue_scalefactor_generation, typename ThreadEpilogueOp::ElementBlockScaleFactor, void>;
|
||||
using LayoutSFD = cute::conditional_t<epilogue_scalefactor_generation, typename ThreadEpilogueOp::GmemLayoutTagScalefactor, LayoutD>;
|
||||
|
||||
|
||||
|
||||
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementA>();
|
||||
|
||||
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementB>();
|
||||
|
||||
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
|
||||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
|
||||
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
|
||||
|
||||
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
|
||||
using RuntimeDataTypeA = typename Operator::CollectiveMainloop::RuntimeDataTypeA;
|
||||
using RuntimeDataTypeB = typename Operator::CollectiveMainloop::RuntimeDataTypeB;
|
||||
|
||||
|
||||
private:
|
||||
BlockScaledGemmDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
BlockScaledGemmUniversal3xOperation(char const *name = "unknown_gemm"):
|
||||
GemmOperation3xBase<Operator_>(name, GemmKind::kUniversal) {
|
||||
description_.kind = OperationKind::kBlockScaledGemm;
|
||||
description_.SFA.element = NumericTypeMap<ElementSFA>::kId;
|
||||
description_.SFA.layout = LayoutTypeID::kRowMajor;
|
||||
description_.SFA.alignment = 128;
|
||||
description_.SFA.log_extent_range = 32;
|
||||
description_.SFA.log_stride_range = 32;
|
||||
|
||||
description_.SFB.element = NumericTypeMap<ElementSFB>::kId;
|
||||
description_.SFB.layout = LayoutTypeID::kRowMajor;
|
||||
description_.SFB.alignment = 128;
|
||||
description_.SFB.log_extent_range = 32;
|
||||
description_.SFB.log_stride_range = 32;
|
||||
|
||||
description_.SFVecSize = SFVecSize;
|
||||
|
||||
description_.SFD = make_TensorDescription<ElementSFD, LayoutSFD>(128);
|
||||
description_.EpilogueSFVecSize = SFD_VectorSize;
|
||||
|
||||
|
||||
description_.name = name;
|
||||
description_.provider = Provider::kCUTLASS;
|
||||
description_.gemm_kind = GemmKind::kUniversal;
|
||||
|
||||
description_.tile_description.threadblock_shape = make_Coord(
|
||||
Operator::ThreadblockShape::kM,
|
||||
Operator::ThreadblockShape::kN,
|
||||
Operator::ThreadblockShape::kK);
|
||||
|
||||
if constexpr (Operator::ArchTag::kMinComputeCapability >= 90) {
|
||||
description_.tile_description.cluster_shape = make_Coord(
|
||||
Operator::ClusterShape::kM,
|
||||
Operator::ClusterShape::kN,
|
||||
Operator::ClusterShape::kK);
|
||||
}
|
||||
|
||||
description_.tile_description.threadblock_stages = Operator::kStages;
|
||||
|
||||
description_.tile_description.warp_count = make_Coord(
|
||||
Operator::WarpCount::kM,
|
||||
Operator::WarpCount::kN,
|
||||
Operator::WarpCount::kK);
|
||||
|
||||
description_.tile_description.math_instruction.instruction_shape = make_Coord(
|
||||
Operator::InstructionShape::kM,
|
||||
Operator::InstructionShape::kN,
|
||||
Operator::InstructionShape::kK);
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.opcode_class =
|
||||
OpcodeClassMap<typename Operator::OperatorClass>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.math_operation =
|
||||
MathOperationMap<typename Operator::MathOperator>::kId;
|
||||
|
||||
description_.tile_description.minimum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMin;
|
||||
|
||||
description_.tile_description.maximum_compute_capability =
|
||||
ArchMap<typename Operator::ArchTag, typename Operator::OperatorClass>::kMax;
|
||||
|
||||
description_.A = make_TensorDescription<ElementA, LayoutA>(Operator::kAlignmentA);
|
||||
description_.B = make_TensorDescription<ElementB, LayoutB>(Operator::kAlignmentB);
|
||||
description_.C = make_TensorDescription<ElementC, LayoutC>(Operator::kAlignmentC);
|
||||
description_.D = make_TensorDescription<ElementD, LayoutD>(Operator::kAlignmentD);
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
description_.split_k_mode = SplitKMode::kNone;
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
BlockScaledGemmDescription const& get_gemm_description() const {
|
||||
return description_;
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status construct_arguments_(
|
||||
OperatorArguments &operator_args, GemmUniversalConfiguration const *configuration) {
|
||||
// NOTE: GemmUniversalConfiguration does not contain problem shapes or batch strides
|
||||
// Do nothing here and construct kernel arguments in update_arguments_ instead
|
||||
// We also cannot construct TMA descriptors without all the arguments available
|
||||
|
||||
operator_args.mode = configuration->mode;
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
template<class FusionArgs, class = void>
|
||||
struct UpdateFusionArgs {
|
||||
static Status update_(FusionArgs const& fusion_args, BlockScaledGemmArguments const &arguments) {
|
||||
// If a custom EVT is instantiated then it is the users's responsibility
|
||||
// to ensure alpha and beta are updated appropriately
|
||||
return Status::kSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
template<class FusionArgs>
|
||||
struct UpdateFusionArgs<FusionArgs, cute::void_t<decltype(FusionArgs{}.alpha)>> {
|
||||
static Status update_(FusionArgs& fusion_args, BlockScaledGemmArguments const &arguments) {
|
||||
|
||||
if constexpr (epilogue_scalefactor_generation) {
|
||||
fusion_args.block_scale_factor_ptr = static_cast<ElementSFD*>(arguments.SFD);
|
||||
fusion_args.norm_constant_ptr = static_cast<ElementCompute const *>(arguments.norm_constant);
|
||||
}
|
||||
|
||||
|
||||
if (arguments.pointer_mode == ScalarPointerMode::kHost) {
|
||||
fusion_args.alpha = *static_cast<ElementCompute const *>(arguments.alpha);
|
||||
fusion_args.beta = *static_cast<ElementCompute const *>(arguments.beta);
|
||||
fusion_args.alpha_ptr = nullptr;
|
||||
fusion_args.beta_ptr = nullptr;
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else if (arguments.pointer_mode == ScalarPointerMode::kDevice) {
|
||||
fusion_args.alpha = 0;
|
||||
fusion_args.beta = 0;
|
||||
fusion_args.alpha_ptr = static_cast<ElementCompute const *>(arguments.alpha);
|
||||
fusion_args.beta_ptr = static_cast<ElementCompute const *>(arguments.beta);
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
else {
|
||||
return Status::kErrorInvalidProblem;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Constructs the arguments structure given the configuration and arguments
|
||||
static Status update_arguments_(
|
||||
OperatorArguments &operator_args,
|
||||
BlockScaledGemmArguments const *arguments) {
|
||||
Status status = Status::kSuccess;
|
||||
|
||||
status = UpdateFusionArgs<decltype(operator_args.epilogue.thread)>::update_(
|
||||
operator_args.epilogue.thread, *arguments);
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
operator_args.problem_shape = cute::make_shape(
|
||||
arguments->problem_size.m(),
|
||||
arguments->problem_size.n(),
|
||||
arguments->problem_size.k(),
|
||||
arguments->batch_count);
|
||||
|
||||
// update arguments
|
||||
|
||||
if constexpr (IsRuntimeDataType) {
|
||||
using ArrayElementA = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementA;
|
||||
using ArrayElementB = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementB;
|
||||
operator_args.mainloop.ptr_A = static_cast<ArrayElementA const *>(arguments->A);
|
||||
operator_args.mainloop.ptr_B = static_cast<ArrayElementB const *>(arguments->B);
|
||||
|
||||
using RuntimeDataTypeA = typename Operator::GemmKernel::CollectiveMainloop::RuntimeDataTypeA;
|
||||
using RuntimeDataTypeB = typename Operator::GemmKernel::CollectiveMainloop::RuntimeDataTypeB;
|
||||
|
||||
static_assert(cute::is_same_v<RuntimeDataTypeA, RuntimeDataTypeB>,
|
||||
"RuntimeDataTypeA/B should be identical, either MXF8F6F4Format or MXF4Format");
|
||||
using RuntimeDatatypeArg = RuntimeDataTypeA;
|
||||
|
||||
auto mapping = [](RuntimeDatatype type) {
|
||||
if constexpr (cute::is_same_v<RuntimeDatatypeArg, cute::UMMA::MXF8F6F4Format>) {
|
||||
if (type == RuntimeDatatype::kE3M2) {
|
||||
return cute::UMMA::MXF8F6F4Format::E3M2;
|
||||
} else if (type == RuntimeDatatype::kE2M3) {
|
||||
return cute::UMMA::MXF8F6F4Format::E2M3;
|
||||
} else if (type == RuntimeDatatype::kE2M1) {
|
||||
return cute::UMMA::MXF8F6F4Format::E2M1;
|
||||
} else {
|
||||
assert("Invalid input datatype.");
|
||||
}
|
||||
}
|
||||
else if constexpr (cute::is_same_v<RuntimeDatatypeArg, cute::UMMA::MXF4Format>) {
|
||||
if (type == RuntimeDatatype::kE2M1) {
|
||||
return cute::UMMA::MXF4Format::E2M1;
|
||||
} else {
|
||||
assert("Invalid input datatype.");
|
||||
}
|
||||
}
|
||||
// BlockScaled kernels receive either MXF4Format or MXF8F6F4Format runtime datatype
|
||||
CUTE_GCC_UNREACHABLE;
|
||||
};
|
||||
|
||||
operator_args.mainloop.runtime_data_type_a = mapping(arguments->runtime_input_datatype_a);
|
||||
operator_args.mainloop.runtime_data_type_b = mapping(arguments->runtime_input_datatype_b);
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
operator_args.mainloop.ptr_A = static_cast<ElementA const *>(arguments->A);
|
||||
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
|
||||
}
|
||||
operator_args.mainloop.ptr_SFA = static_cast<ElementSFA const *>(arguments->SFA);
|
||||
operator_args.mainloop.ptr_SFB = static_cast<ElementSFB const *>(arguments->SFB);
|
||||
operator_args.epilogue.ptr_C = static_cast<ElementC const *>(arguments->C);
|
||||
operator_args.epilogue.ptr_D = static_cast<ElementD *>(arguments->D);
|
||||
|
||||
operator_args.mainloop.dA = cute::make_int_tuple_from<typename Operator::GemmKernel::StrideA>(
|
||||
arguments->lda, arguments->batch_stride_A);
|
||||
operator_args.mainloop.dB = cute::make_int_tuple_from<typename Operator::GemmKernel::StrideB>(
|
||||
arguments->ldb, arguments->batch_stride_B);
|
||||
operator_args.epilogue.dC = cute::make_int_tuple_from<typename Operator::GemmKernel::StrideC>(
|
||||
arguments->ldc, arguments->batch_stride_C);
|
||||
operator_args.epilogue.dD = operator_args.epilogue.dC;
|
||||
|
||||
operator_args.mainloop.layout_SFA = Sm100BlkScaledConfig::tile_atom_to_shape_SFA(operator_args.problem_shape);
|
||||
operator_args.mainloop.layout_SFB = Sm100BlkScaledConfig::tile_atom_to_shape_SFB(operator_args.problem_shape);
|
||||
|
||||
/* Query device SM count to pass onto the kernel as an argument, where needed */
|
||||
operator_args.hw_info.sm_count = arguments->sm_count;
|
||||
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.max_swizzle_size)>) {
|
||||
operator_args.scheduler.max_swizzle_size = arguments->swizzle_size;
|
||||
}
|
||||
|
||||
if constexpr (!std::is_const_v<decltype(operator_args.scheduler.raster_order)>) {
|
||||
using Enum_t = decltype(operator_args.scheduler.raster_order);
|
||||
switch (arguments->raster_order) {
|
||||
case RasterOrder::kAlongN:
|
||||
operator_args.scheduler.raster_order = Enum_t::AlongN;
|
||||
break;
|
||||
case RasterOrder::kAlongM:
|
||||
operator_args.scheduler.raster_order = Enum_t::AlongM;
|
||||
break;
|
||||
default:
|
||||
operator_args.scheduler.raster_order = Enum_t::Heuristic;
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<typename Operator::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
|
||||
operator_args.scheduler.splits = arguments->split_k_slices;
|
||||
}
|
||||
|
||||
|
||||
if constexpr (Operator::ArchTag::kMinComputeCapability >= 100) {
|
||||
operator_args.hw_info.cluster_shape = dim3(
|
||||
arguments->cluster_shape.m(),
|
||||
arguments->cluster_shape.n(),
|
||||
arguments->cluster_shape.k());
|
||||
operator_args.hw_info.cluster_shape_fallback = dim3(
|
||||
arguments->cluster_shape_fallback.m(),
|
||||
arguments->cluster_shape_fallback.n(),
|
||||
arguments->cluster_shape_fallback.k());
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
/// Returns success if the operation can proceed
|
||||
Status can_implement(
|
||||
void const *configuration_ptr, void const *arguments_ptr) const override {
|
||||
|
||||
GemmUniversalConfiguration const *configuration =
|
||||
static_cast<GemmUniversalConfiguration const *>(configuration_ptr);
|
||||
BlockScaledGemmArguments const *arguments =
|
||||
static_cast<BlockScaledGemmArguments const *>(arguments_ptr);
|
||||
|
||||
OperatorArguments args;
|
||||
auto status = update_arguments_(args, arguments);
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// can_implement rules may need access to problem shape
|
||||
args.problem_shape = cute::make_shape(
|
||||
configuration->problem_size.m(),
|
||||
configuration->problem_size.n(),
|
||||
configuration->problem_size.k(),
|
||||
configuration->batch_count);
|
||||
|
||||
return Operator::can_implement(args);
|
||||
}
|
||||
|
||||
/// Gets the host-side workspace
|
||||
uint64_t get_host_workspace_size(void const *configuration) const override {
|
||||
return sizeof(Operator);
|
||||
}
|
||||
|
||||
/// Gets the device-side workspace
|
||||
uint64_t get_device_workspace_size(
|
||||
void const *configuration_ptr,void const *arguments_ptr) const override {
|
||||
|
||||
OperatorArguments args;
|
||||
auto status = update_arguments_(
|
||||
args, static_cast<BlockScaledGemmArguments const *>(arguments_ptr));
|
||||
if (status != Status::kSuccess) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t size = Operator::get_workspace_size(args);
|
||||
return size;
|
||||
}
|
||||
|
||||
/// Initializes the workspace
|
||||
Status initialize(
|
||||
void const *configuration_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace,
|
||||
cudaStream_t stream = nullptr) const override {
|
||||
Operator *op = new (host_workspace) Operator;
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
Status initialize_with_profiler_workspace(
|
||||
void const *configuration,
|
||||
void *host_workspace,
|
||||
void *device_workspace,
|
||||
uint8_t **profiler_workspaces,
|
||||
int problem_count_from_profiler,
|
||||
cudaStream_t stream = nullptr) {
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
/// Runs the kernel
|
||||
Status run(
|
||||
void const *arguments_ptr,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const override {
|
||||
|
||||
OperatorArguments args;
|
||||
Status status = update_arguments_(args, static_cast<BlockScaledGemmArguments const *>(arguments_ptr));
|
||||
if (status != Status::kSuccess) {
|
||||
return status;
|
||||
}
|
||||
|
||||
Operator *op = static_cast<Operator *>(host_workspace);
|
||||
// We need to call initialize() since we have to rebuild TMA desc for every new set of args
|
||||
status = op->run(args, device_workspace, stream, nullptr, static_cast<BlockScaledGemmArguments const *>(arguments_ptr)->use_pdl);
|
||||
return status;
|
||||
}
|
||||
};
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace cutlass::library
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -162,6 +162,18 @@ public:
|
||||
using CollectiveEpilogue = typename Operator::CollectiveEpilogue;
|
||||
using ThreadEpilogueOp = typename CollectiveEpilogue::ThreadEpilogueOp;
|
||||
|
||||
|
||||
static constexpr bool IsRuntimeDataTypeA = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementA>();
|
||||
|
||||
static constexpr bool IsRuntimeDataTypeB = cutlass::gemm::collective::detail::is_sm10x_runtime_f8f6f4<ElementB>();
|
||||
|
||||
static_assert((IsRuntimeDataTypeA && IsRuntimeDataTypeB) ||
|
||||
(!IsRuntimeDataTypeA && !IsRuntimeDataTypeB),
|
||||
"ElementA and ElementB in a GEMM kernel should be both runtime or both static.");
|
||||
|
||||
static constexpr bool IsRuntimeDataType = IsRuntimeDataTypeA && IsRuntimeDataTypeB;
|
||||
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
@@ -235,8 +247,42 @@ protected:
|
||||
arguments->batch_count);
|
||||
|
||||
// update arguments
|
||||
|
||||
|
||||
if constexpr (IsRuntimeDataType) {
|
||||
using ArrayElementA = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementA;
|
||||
using ArrayElementB = typename Operator::GemmKernel::CollectiveMainloop::ArrayElementB;
|
||||
operator_args.mainloop.ptr_A = static_cast<ArrayElementA const *>(arguments->A);
|
||||
operator_args.mainloop.ptr_B = static_cast<ArrayElementB const *>(arguments->B);
|
||||
|
||||
std::unordered_map<RuntimeDatatype, cute::UMMA::MXF8F6F4Format> mapping = {
|
||||
{RuntimeDatatype::kE4M3, cute::UMMA::MXF8F6F4Format::E4M3},
|
||||
{RuntimeDatatype::kE5M2, cute::UMMA::MXF8F6F4Format::E5M2},
|
||||
{RuntimeDatatype::kE3M2, cute::UMMA::MXF8F6F4Format::E3M2},
|
||||
{RuntimeDatatype::kE2M1, cute::UMMA::MXF8F6F4Format::E2M1}
|
||||
};
|
||||
|
||||
auto iter_runtime_a = mapping.find(arguments->runtime_input_datatype_a);
|
||||
auto iter_runtime_b = mapping.find(arguments->runtime_input_datatype_b);
|
||||
|
||||
if (iter_runtime_a != mapping.end()) {
|
||||
operator_args.mainloop.runtime_data_type_a = iter_runtime_a->second;
|
||||
} else {
|
||||
assert("invalid runtime argument for datatype A!");
|
||||
}
|
||||
|
||||
if (iter_runtime_b != mapping.end()) {
|
||||
operator_args.mainloop.runtime_data_type_b = iter_runtime_b->second;
|
||||
} else {
|
||||
assert("invalid runtime argument for datatype B!");
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
operator_args.mainloop.ptr_A = static_cast<ElementA const *>(arguments->A);
|
||||
operator_args.mainloop.ptr_B = static_cast<ElementB const *>(arguments->B);
|
||||
}
|
||||
operator_args.epilogue.ptr_C = static_cast<ElementC const *>(arguments->C);
|
||||
operator_args.epilogue.ptr_D = static_cast<ElementD *>(arguments->D);
|
||||
|
||||
@@ -277,6 +323,22 @@ protected:
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<typename Operator::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
|
||||
operator_args.scheduler.splits = arguments->split_k_slices;
|
||||
}
|
||||
|
||||
|
||||
if constexpr (Operator::ArchTag::kMinComputeCapability >= 100) {
|
||||
operator_args.hw_info.cluster_shape = dim3(
|
||||
arguments->cluster_shape.m(),
|
||||
arguments->cluster_shape.n(),
|
||||
arguments->cluster_shape.k());
|
||||
operator_args.hw_info.cluster_shape_fallback = dim3(
|
||||
arguments->cluster_shape_fallback.m(),
|
||||
arguments->cluster_shape_fallback.n(),
|
||||
arguments->cluster_shape_fallback.k());
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
|
||||
@@ -510,6 +510,15 @@ Status Handle::gemm_universal(
|
||||
int M, /// GEMM M dimension
|
||||
int N, /// GEMM N dimension
|
||||
int K, /// GEMM K dimension
|
||||
|
||||
int cluster_m, /// cluster shape M dimension
|
||||
int cluster_n, /// cluster shape N dimension
|
||||
int cluster_k, /// cluster shape K dimension
|
||||
int cluster_m_fallback, /// Fallback cluster shape M dimension
|
||||
int cluster_n_fallback, /// Fallback cluster shape N dimension
|
||||
int cluster_k_fallback, /// Fallback cluster shape K dimension
|
||||
|
||||
|
||||
NumericTypeID element_compute, /// Data type of internal accumulation
|
||||
|
||||
NumericTypeID element_scalar, /// Data type of alpha/beta scalars
|
||||
@@ -629,6 +638,8 @@ Status Handle::gemm_universal(
|
||||
GemmUniversalConfiguration configuration{
|
||||
mode,
|
||||
{M, N, K},
|
||||
{cluster_m, cluster_n, cluster_k},
|
||||
{cluster_m_fallback, cluster_n_fallback, cluster_k_fallback},
|
||||
batch_count,
|
||||
lda,
|
||||
ldb,
|
||||
@@ -647,6 +658,8 @@ Status Handle::gemm_universal(
|
||||
|
||||
GemmUniversalArguments arguments{
|
||||
{M, N, K},
|
||||
{cluster_m, cluster_n, cluster_k},
|
||||
{cluster_m_fallback, cluster_n_fallback, cluster_k_fallback},
|
||||
batch_count,
|
||||
ptr_A,
|
||||
ptr_B,
|
||||
|
||||
@@ -116,6 +116,27 @@ template <> struct NumericTypeMap<cutlass::float_e5m2_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFE5M2;
|
||||
};
|
||||
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::float_e2m3_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFE2M3;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::float_e3m2_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFE3M2;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::float_e2m1_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFE2M1;
|
||||
};
|
||||
template <> struct NumericTypeMap<cutlass::float_ue8m0_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFUE8M0;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::float_ue4m3_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kFUE4M3;
|
||||
};
|
||||
|
||||
|
||||
template <> struct NumericTypeMap<uint16_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kU16;
|
||||
};
|
||||
@@ -161,6 +182,21 @@ template <> struct NumericTypeMap<cutlass::tfloat32_t> {
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::type_erased_dynamic_float8_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kF8;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::type_erased_dynamic_float6_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kF6;
|
||||
};
|
||||
|
||||
template <> struct NumericTypeMap<cutlass::type_erased_dynamic_float4_t> {
|
||||
static NumericTypeID const kId = NumericTypeID::kF4;
|
||||
};
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename T> struct MathOperationMap {
|
||||
@@ -300,6 +336,12 @@ template <> struct OpcodeClassMap<arch::OpClassSparseTensorOp> {
|
||||
static OpcodeClassID const kId = OpcodeClassID::kSparseTensorOp;
|
||||
};
|
||||
|
||||
|
||||
template <> struct OpcodeClassMap<arch::OpClassBlockScaledTensorOp> {
|
||||
static OpcodeClassID const kId = OpcodeClassID::kBlockScaledOp;
|
||||
};
|
||||
|
||||
|
||||
template <> struct OpcodeClassMap<arch::OpClassWmmaTensorOp> {
|
||||
static OpcodeClassID const kId = OpcodeClassID::kWmmaTensorOp;
|
||||
};
|
||||
|
||||
@@ -48,6 +48,45 @@ void OperationTable::append(Manifest const &manifest) {
|
||||
// Insert operations into appropriate data structure
|
||||
for (auto const & operation : manifest) {
|
||||
OperationDescription const &desc = operation->description();
|
||||
|
||||
if (desc.kind == OperationKind::kBlockScaledGemm) {
|
||||
BlockScaledGemmDescription const &gemm_desc = static_cast<BlockScaledGemmDescription const &>(desc);
|
||||
|
||||
BlockScaledGemmFunctionalKey functional_key(
|
||||
gemm_desc.provider,
|
||||
gemm_desc.gemm_kind,
|
||||
gemm_desc.kind,
|
||||
gemm_desc.tile_description.math_instruction.element_accumulator,
|
||||
gemm_desc.element_epilogue,
|
||||
gemm_desc.A.element,
|
||||
gemm_desc.A.layout,
|
||||
gemm_desc.SFA.element,
|
||||
gemm_desc.B.element,
|
||||
gemm_desc.B.layout,
|
||||
gemm_desc.SFB.element,
|
||||
gemm_desc.C.element,
|
||||
gemm_desc.C.layout,
|
||||
gemm_desc.D.element,
|
||||
gemm_desc.D.layout,
|
||||
gemm_desc.SFD.element,
|
||||
gemm_desc.SFD.layout,
|
||||
gemm_desc.SFVecSize
|
||||
, gemm_desc.EpilogueSFVecSize
|
||||
);
|
||||
|
||||
Operation const *op = operation.get();
|
||||
|
||||
int cc = gemm_desc.tile_description.minimum_compute_capability;
|
||||
|
||||
int alignment = std::max(std::max(
|
||||
gemm_desc.A.alignment, gemm_desc.B.alignment), gemm_desc.C.alignment);
|
||||
|
||||
GemmPreferenceKey preference_key(cc, alignment);
|
||||
|
||||
block_scaled_gemm_operations[functional_key][preference_key].push_back(op);
|
||||
}
|
||||
|
||||
|
||||
// insert all gemm operation into operation table
|
||||
if (desc.kind == OperationKind::kGemm) {
|
||||
GemmDescription const &gemm_desc = static_cast<GemmDescription const &>(desc);
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "block_scaled_gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_block_scaled_gemm_reference_operations_fp4a_vs16(Manifest &manifest) {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// SFVectorSize = 16 with MxF4NvF4 instructions
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// (float_e2m1_t * float_ue4m3_t) * (float_e2m1_t * float_ue4m3_t)
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
16 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue4m3_t /*SFA*/, float_e2m1_t /*B*/, float_ue4m3_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
// (float_e2m1_t * float_ue8m0_t) * (float_e2m1_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 16 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
16 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
16 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm_tn<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 16 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,130 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "block_scaled_gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_block_scaled_gemm_reference_operations_fp4a_vs32(Manifest &manifest) {
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// SFVectorSize = 32 with MxF4 instructions
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// (float_e2m1_t * float_ue8m0_t) * (float_e2m1_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// With SF generation reference
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 32 /*SFVecSize*/,
|
||||
16 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 32 /*SFVecSize*/,
|
||||
16 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e2m1_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpiSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,354 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "block_scaled_gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void initialize_block_scaled_gemm_reference_operations_mixed8bitsa(Manifest &manifest) {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// SFVectorSize = 32 with MxF8F6F4 instructions
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// (float_e2m3_t * float_ue8m0_t) * (float_e2m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// (float_e4m3_t * float_ue8m0_t) * (float_e2m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// (float_e2m3_t * float_ue8m0_t) * (float_e4m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// (float_e2m1_t * float_ue8m0_t) * (float_e4m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// (float_e4m3_t * float_ue8m0_t) * (float_e2m1_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
// (float_e4m3_t * float_ue8m0_t) * (float_e4m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e4m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e4m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
|
||||
// (float_e3m2_t * float_ue8m0_t) * (float_e2m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e3m2_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e3m2_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e3m2_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e3m2_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e3m2_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
// (float_e2m1_t * float_ue8m0_t) * (float_e2m3_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m1_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m3_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
|
||||
// (float_e2m3_t * float_ue8m0_t) * (float_e2m1_t * float_ue8m0_t)
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
void /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, void /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e3m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
make_block_scaled_gemm<
|
||||
float_e2m3_t /*A*/, float_ue8m0_t /*SFA*/, float_e2m1_t /*B*/, float_ue8m0_t /*SFB*/,
|
||||
half_t /*C*/, float /*Compute*/, float_ue8m0_t /*SFD*/, float /*Accum*/, float_e5m2_t /*D*/, 32 /*SFVecSize*/,
|
||||
32 /*EpilogueSFVecSize*/
|
||||
>(manifest);
|
||||
|
||||
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,459 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Defines reference operations for block-scaled GEMM operation kinds in CUTLASS Library
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <cstring>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
#include "cutlass/library/util.h"
|
||||
#include "cutlass/util/packed_stride.hpp"
|
||||
#include "library_internal.h"
|
||||
|
||||
#include "cutlass/util/reference/host/gett.hpp"
|
||||
#include "cutlass/detail/sm100_blockscaled_layout.hpp"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
auto make_iterator(T* ptr) {
|
||||
using namespace cute;
|
||||
if constexpr (cute::is_subbyte_v<T>) {
|
||||
return subbyte_iterator<T>(ptr);
|
||||
}
|
||||
else {
|
||||
return ptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
Provider Provider_,
|
||||
typename ElementA_,
|
||||
typename LayoutA_,
|
||||
typename ElementSFA_,
|
||||
typename ElementB_,
|
||||
typename LayoutB_,
|
||||
typename ElementSFB_,
|
||||
typename ElementC_,
|
||||
typename LayoutC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ElementD_ = ElementC_,
|
||||
typename ElementSFD_ = void,
|
||||
typename LayoutSFD_ = LayoutC_,
|
||||
int SFVecSize_ = 32,
|
||||
int EpilogueSFVecSize_ = 0,
|
||||
typename ConvertOp_ = NumericConverter<ElementD_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
class BlockScaledGemmReferenceOperation : public Operation {
|
||||
public:
|
||||
static Provider const kProvider = Provider_;
|
||||
|
||||
using ElementA = ElementA_;
|
||||
using LayoutA = LayoutA_;
|
||||
using ElementSFA = ElementSFA_;
|
||||
using ElementB = ElementB_;
|
||||
using LayoutB = LayoutB_;
|
||||
using ElementSFB = ElementSFB_;
|
||||
using ElementC = ElementC_;
|
||||
using LayoutC = LayoutC_;
|
||||
using ElementD = ElementD_;
|
||||
using ElementSFD = ElementSFD_;
|
||||
using LayoutSFD = LayoutSFD_;
|
||||
using ElementCompute = ElementCompute_;
|
||||
using ElementAccumulator = ElementAccumulator_;
|
||||
using ConvertOp = ConvertOp_;
|
||||
using InnerProductOp = InnerProductOp_;
|
||||
constexpr static int SFVecSize = SFVecSize_;
|
||||
constexpr static int EpilogueSFVecSize = EpilogueSFVecSize_;
|
||||
|
||||
protected:
|
||||
|
||||
/// Storage for the name string
|
||||
std::string name_;
|
||||
|
||||
///
|
||||
BlockScaledGemmDescription description_;
|
||||
|
||||
public:
|
||||
|
||||
/// Constructor
|
||||
BlockScaledGemmReferenceOperation() {
|
||||
|
||||
// Basic information
|
||||
description_.provider = kProvider;
|
||||
description_.kind = OperationKind::kBlockScaledGemm;
|
||||
description_.gemm_kind = GemmKind::kUniversal;
|
||||
|
||||
// Tensor description
|
||||
description_.A = make_TensorDescription<ElementA, LayoutA>();
|
||||
description_.SFA = make_TensorDescription<ElementSFA, LayoutA>();
|
||||
description_.B = make_TensorDescription<ElementB, LayoutB>();
|
||||
description_.SFB = make_TensorDescription<ElementSFB, LayoutB>();
|
||||
description_.C = make_TensorDescription<ElementC, LayoutC>();
|
||||
description_.D = make_TensorDescription<ElementD, LayoutC>();
|
||||
description_.SFD = make_TensorDescription<ElementSFD, LayoutSFD>();
|
||||
|
||||
// Epilogue compute and accumulator type description
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
|
||||
// Compute capability for gemm reference
|
||||
description_.tile_description.minimum_compute_capability =
|
||||
(kProvider == Provider::kReferenceDevice ? 50 : 0);
|
||||
|
||||
description_.tile_description.maximum_compute_capability = 1024;
|
||||
|
||||
description_.SFVecSize = SFVecSize;
|
||||
description_.EpilogueSFVecSize = EpilogueSFVecSize;
|
||||
|
||||
// Procedural name
|
||||
std::stringstream ss;
|
||||
|
||||
ss << "gemm"
|
||||
<< "_reference_" << to_string(description_.provider)
|
||||
<< "_" << to_string(description_.A.element) << to_string(description_.A.layout)
|
||||
<< "_" << to_string(description_.SFA.element) << to_string(description_.SFA.layout)
|
||||
<< "_" << to_string(description_.B.element) << to_string(description_.B.layout)
|
||||
<< "_" << to_string(description_.SFB.element) << to_string(description_.SFB.layout)
|
||||
<< "_" << to_string(description_.C.element) << to_string(description_.C.layout)
|
||||
<< "_" << to_string(description_.SFD.element) << to_string(description_.SFD.layout)
|
||||
<< "_" << to_string(description_.tile_description.math_instruction.element_accumulator);
|
||||
|
||||
name_ = ss.str();
|
||||
|
||||
description_.name = name_.c_str();
|
||||
|
||||
// Epilogue compute and accumulator type description
|
||||
description_.element_epilogue = NumericTypeMap<ElementCompute>::kId;
|
||||
|
||||
description_.tile_description.math_instruction.element_accumulator =
|
||||
NumericTypeMap<ElementAccumulator>::kId;
|
||||
}
|
||||
|
||||
/// Returns the description of the GEMM operation
|
||||
virtual OperationDescription const & description() const {
|
||||
return description_;
|
||||
}
|
||||
|
||||
virtual Status can_implement(
|
||||
void const *configuration,
|
||||
void const *arguments) const {
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
virtual uint64_t get_host_workspace_size(
|
||||
void const *configuration) const {
|
||||
|
||||
return sizeof(GemmUniversalConfiguration);
|
||||
}
|
||||
|
||||
virtual uint64_t get_device_workspace_size(
|
||||
void const *configuration,
|
||||
void const *arguments = nullptr) const {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual Status initialize(
|
||||
void const *configuration,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
return Status::kSuccess;
|
||||
}
|
||||
|
||||
virtual Status run(
|
||||
void const *arguments,
|
||||
void *host_workspace,
|
||||
void *device_workspace = nullptr,
|
||||
cudaStream_t stream = nullptr) const {
|
||||
using namespace cute;
|
||||
|
||||
BlockScaledGemmArguments const &args = *static_cast<BlockScaledGemmArguments const *>(arguments);
|
||||
|
||||
// Construct cute::Tensor A/B/C
|
||||
|
||||
int M = args.problem_size.m();
|
||||
int N = args.problem_size.n();
|
||||
int K = args.problem_size.k();
|
||||
int L = args.batch_count;
|
||||
|
||||
auto problem_shape_MNKL = cute::make_shape(M, N, K, L);
|
||||
|
||||
auto alpha = *(static_cast<ElementCompute const*>(args.alpha));
|
||||
auto beta = *(static_cast<ElementCompute const*>(args.beta));
|
||||
|
||||
using StrideA = cutlass::gemm::TagToStrideA_t<LayoutA>;
|
||||
using StrideB = cutlass::gemm::TagToStrideB_t<LayoutB>;
|
||||
using StrideC = cutlass::gemm::TagToStrideC_t<LayoutC>;
|
||||
using StrideD = cutlass::gemm::TagToStrideC_t<LayoutC>;
|
||||
|
||||
auto stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
|
||||
auto stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(N, K, L));
|
||||
auto stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
|
||||
auto stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
|
||||
|
||||
using Sm100BlockScaledConfig = cutlass::detail::Sm100BlockScaledConfig<SFVecSize>;
|
||||
auto A = cute::make_tensor(detail::make_iterator(static_cast<ElementA const*>(args.A)),
|
||||
cute::make_layout(cute::make_shape(M, K, L), stride_a));
|
||||
auto SfA = make_tensor(static_cast<ElementSFA const*>(args.SFA), Sm100BlockScaledConfig::tile_atom_to_shape_SFA(problem_shape_MNKL));
|
||||
|
||||
auto B = cute::make_tensor(detail::make_iterator(static_cast<ElementB const*>(args.B)),
|
||||
cute::make_layout(cute::make_shape(N, K, L), stride_b));
|
||||
auto SfB = make_tensor(static_cast<ElementSFB const*>(args.SFB), Sm100BlockScaledConfig::tile_atom_to_shape_SFB(problem_shape_MNKL));
|
||||
|
||||
auto C = [&]() {
|
||||
if constexpr (not is_same_v<ElementC, void>) {
|
||||
return cute::make_tensor(detail::make_iterator(static_cast<ElementC const*>(args.C)),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_c));
|
||||
}
|
||||
else {
|
||||
return cute::make_tensor(detail::make_iterator(static_cast<ElementD const*>(nullptr)),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_c));
|
||||
}
|
||||
}();
|
||||
|
||||
auto D = cute::make_tensor(detail::make_iterator(static_cast<ElementD *>(args.D)),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_d));
|
||||
|
||||
cutlass::reference::host::GettBlockScalingMainloopParams<ElementAccumulator,
|
||||
decltype(A), decltype(SfA),
|
||||
decltype(B), decltype(SfB)>
|
||||
mainloop_params{A, SfA, B, SfB};
|
||||
|
||||
if constexpr (not is_same_v<ElementSFD, void>) {
|
||||
|
||||
using Sm100BlockScaledOutputConfig = cutlass::detail::Sm100BlockScaledOutputConfig<
|
||||
EpilogueSFVecSize
|
||||
>;
|
||||
|
||||
auto SfD = cute::make_tensor(detail::make_iterator(static_cast<ElementSFD*>(args.SFD)), Sm100BlockScaledOutputConfig::tile_atom_to_shape_SFD(problem_shape_MNKL));
|
||||
|
||||
cutlass::reference::host::GettBlockScalingEpilogueParams<
|
||||
ElementCompute, ElementAccumulator, ElementCompute,
|
||||
decltype(C), decltype(D), decltype(SfD), Int<EpilogueSFVecSize>, cutlass::reference::host::SfStrategy::SfDGen>
|
||||
epilogue_params{alpha, beta, C, D, SfD, *(static_cast<ElementCompute const*>(args.norm_constant))};
|
||||
|
||||
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
|
||||
}
|
||||
else {
|
||||
// W/O SF generation
|
||||
auto SfD = cute::make_tensor(static_cast<ElementSFA *>(nullptr),
|
||||
cute::make_layout(cute::make_shape(M, N, L))); // not used.
|
||||
cutlass::reference::host::GettBlockScalingEpilogueParams<
|
||||
ElementCompute, ElementAccumulator, ElementCompute,
|
||||
decltype(C), decltype(D), decltype(SfD)>
|
||||
epilogue_params{alpha, beta, C, D, SfD};
|
||||
|
||||
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
|
||||
}
|
||||
|
||||
return Status::kSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename ElementA_,
|
||||
typename ElementSFA_,
|
||||
typename ElementB_,
|
||||
typename ElementSFB_,
|
||||
typename ElementC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementSFD_ = void,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ElementD_ = ElementC_,
|
||||
int SFVecSize = 32,
|
||||
int EpilogueSFVecSize = SFVecSize,
|
||||
typename ConvertOp_ = NumericConverter<ElementD_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
void make_block_scaled_gemm_tn(Manifest &manifest) {
|
||||
#if !defined(CUTLASS_PROFILER_DISABLE_REFERENCE)
|
||||
manifest.append(new BlockScaledGemmReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
ElementA_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementSFA_,
|
||||
ElementB_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementSFB_,
|
||||
ElementC_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ElementD_,
|
||||
ElementSFD_,
|
||||
cutlass::layout::RowMajor,
|
||||
SFVecSize,
|
||||
EpilogueSFVecSize,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
#endif // !defined(CUTLASS_PROFILER_DISABLE_REFERENCE)
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename ElementA_,
|
||||
typename ElementSFA_,
|
||||
typename ElementB_,
|
||||
typename ElementSFB_,
|
||||
typename ElementC_,
|
||||
typename ElementCompute_,
|
||||
typename ElementSFD_ = void,
|
||||
typename ElementAccumulator_ = ElementCompute_,
|
||||
typename ElementD_ = ElementC_,
|
||||
int SFVecSize = 32,
|
||||
int EpilogueSFVecSize = SFVecSize,
|
||||
typename ConvertOp_ = NumericConverter<ElementD_, ElementCompute_>,
|
||||
typename InnerProductOp_ = multiply_add<ElementAccumulator_>
|
||||
>
|
||||
void make_block_scaled_gemm(Manifest &manifest) {
|
||||
///
|
||||
/// A is Row , B is Col
|
||||
///
|
||||
manifest.append(new BlockScaledGemmReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
ElementA_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementSFA_,
|
||||
ElementB_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementSFB_,
|
||||
ElementC_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ElementD_,
|
||||
ElementSFD_,
|
||||
cutlass::layout::RowMajor,
|
||||
SFVecSize,
|
||||
EpilogueSFVecSize,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
manifest.append(new BlockScaledGemmReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
ElementA_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementSFA_,
|
||||
ElementB_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementSFB_,
|
||||
ElementC_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ElementD_,
|
||||
ElementSFD_,
|
||||
cutlass::layout::RowMajor,
|
||||
SFVecSize,
|
||||
EpilogueSFVecSize,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
///
|
||||
/// A is Col , B is Row
|
||||
///
|
||||
manifest.append(new BlockScaledGemmReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
ElementA_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementSFA_,
|
||||
ElementB_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementSFB_,
|
||||
ElementC_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ElementD_,
|
||||
ElementSFD_,
|
||||
cutlass::layout::RowMajor,
|
||||
SFVecSize,
|
||||
EpilogueSFVecSize,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
manifest.append(new BlockScaledGemmReferenceOperation<
|
||||
Provider::kReferenceHost,
|
||||
ElementA_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementSFA_,
|
||||
ElementB_,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementSFB_,
|
||||
ElementC_,
|
||||
cutlass::layout::ColumnMajor,
|
||||
ElementCompute_,
|
||||
ElementAccumulator_,
|
||||
ElementD_,
|
||||
ElementSFD_,
|
||||
cutlass::layout::RowMajor,
|
||||
SFVecSize,
|
||||
EpilogueSFVecSize,
|
||||
ConvertOp_,
|
||||
InnerProductOp_
|
||||
>);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A/B : float_e2m1_t (not support float_e0m2_t to reduce ref kernel compile time)
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e2m1_e2m1_f32_f16_e4m3
|
||||
// 2. e2m1_e2m1_f32_f16_e5m2
|
||||
// 3. e2m1_e2m1_f32_f16_f16
|
||||
// 4. e2m1_e2m1_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f4_f4_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e2m1_t
|
||||
// B: float_e3m2_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e2m1_e3m2_f32_f16_e4m3
|
||||
// 2. e2m1_e3m2_f32_f16_e5m2
|
||||
// 3. e2m1_e3m2_f32_f16_f16
|
||||
// 4. e2m1_e3m2_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f4_f6_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e2m1_t
|
||||
// B: float_e4m3_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e2m1_e4m3_f32_f16_e4m3
|
||||
// 2. e2m1_e4m3_f32_f16_e5m2
|
||||
// 3. e2m1_e4m3_f32_f16_f16
|
||||
// 4. e2m1_e4m3_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e2m1_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e3m2_t
|
||||
// B: float_e2m1_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e3m2_e2m1_f32_f16_e4m3
|
||||
// 2. e3m2_e2m1_f32_f16_e5m2
|
||||
// 3. e3m2_e2m1_f32_f16_f16
|
||||
// 4. e3m2_e2m1_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f6_f4_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A/B : float_e3m2_t (not support float_e2m3_t to reduce ref kernel compile time)
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e3m2_e3m2_f32_f16_e4m3
|
||||
// 2. e3m2_e3m2_f32_f16_e5m2
|
||||
// 3. e3m2_e3m2_f32_f16_f16
|
||||
// 4. e3m2_e3m2_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f6_f6_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e3m2_t
|
||||
// B: float_e4m3_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e3m2_e4m3_f32_f16_e4m3
|
||||
// 2. e3m2_e4m3_f32_f16_e5m2
|
||||
// 3. e3m2_e4m3_f32_f16_f16
|
||||
// 4. e3m2_e4m3_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f6_f8_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e3m2_t, // ElementA
|
||||
float_e4m3_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e4m3_t
|
||||
// B: float_e2m1_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e4m3_e2m1_f32_f16_e4m3
|
||||
// 2. e4m3_e2m1_f32_f16_e5m2
|
||||
// 3. e4m3_e2m1_f32_f16_f16
|
||||
// 4. e4m3_e2m1_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f8_f4_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e2m1_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
\brief Instantiates GEMM reference implementations for FP8.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/library/library.h"
|
||||
#include "cutlass/library/manifest.h"
|
||||
|
||||
#include "gemm_reference_operation.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
namespace library {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A: float_e4m3_t
|
||||
// B: float_e3m2_t
|
||||
// Acc: f32
|
||||
// C/D : some variance
|
||||
|
||||
// 1. e4m3_e3m2_f32_f16_e4m3
|
||||
// 2. e4m3_e3m2_f32_f16_e5m2
|
||||
// 3. e4m3_e3m2_f32_f16_f16
|
||||
// 4. e4m3_e3m2_f32_f32_f32
|
||||
|
||||
void initialize_gemm_reference_operations_f8_f6_f32(Manifest &manifest) {
|
||||
|
||||
// 1.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e4m3_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 2.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float_e5m2_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 3.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
half_t, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
half_t // ElementD
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
float_e4m3_t, // ElementA
|
||||
float_e3m2_t, // ElementB
|
||||
float, // ElementC
|
||||
float, // ElementScalar
|
||||
float, // ElementAccumulator
|
||||
float // ElementD
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
} // namespace cutlass
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -87,6 +87,17 @@ void initialize_gemm_reference_operations_u8_u8_s32(Manifest &manifest) {
|
||||
NumericConverterClamp<int8_t, float> // From Scalar to D
|
||||
>(manifest);
|
||||
|
||||
// 4.
|
||||
make_gemm_real_canonical_layouts<
|
||||
uint8_t, // ElementA
|
||||
uint8_t, // ElementB
|
||||
int8_t, // ElementC
|
||||
float, // ElementScalar / ElementCompute
|
||||
int32_t, // ElementAccumulator
|
||||
uint8_t, // ElementD
|
||||
NumericConverterClamp<uint8_t, float> // From Scalar to D
|
||||
>(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -52,6 +52,19 @@ void initialize_gemm_reference_operations_e4m3a_e4m3out(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_e5m2a_e4m3out(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_e4m3a_e5m2out(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_e5m2a_e5m2out(Manifest &manifest);
|
||||
|
||||
void initialize_gemm_reference_operations_f4_f4_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f4_f6_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f4_f8_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f6_f4_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f6_f6_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f6_f8_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f8_f4_f32(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_f8_f6_f32(Manifest &manifest);
|
||||
void initialize_block_scaled_gemm_reference_operations_fp4a_vs16(Manifest &manifest);
|
||||
void initialize_block_scaled_gemm_reference_operations_fp4a_vs32(Manifest &manifest);
|
||||
void initialize_block_scaled_gemm_reference_operations_mixed8bitsa(Manifest &manifest);
|
||||
|
||||
void initialize_gemm_reference_operations_fp8in_fp16out(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_fp8in_bf16out(Manifest &manifest);
|
||||
void initialize_gemm_reference_operations_fp8in_fp32out(Manifest &manifest);
|
||||
@@ -89,6 +102,19 @@ void initialize_reference_operations(Manifest &manifest) {
|
||||
initialize_gemm_reference_operations_fp_mixed_input(manifest);
|
||||
initialize_gemm_reference_operations_int_mixed_input(manifest);
|
||||
|
||||
|
||||
initialize_gemm_reference_operations_f4_f4_f32(manifest);
|
||||
initialize_gemm_reference_operations_f4_f6_f32(manifest);
|
||||
initialize_gemm_reference_operations_f4_f8_f32(manifest);
|
||||
initialize_gemm_reference_operations_f6_f4_f32(manifest);
|
||||
initialize_gemm_reference_operations_f6_f6_f32(manifest);
|
||||
initialize_gemm_reference_operations_f6_f8_f32(manifest);
|
||||
initialize_gemm_reference_operations_f8_f4_f32(manifest);
|
||||
initialize_gemm_reference_operations_f8_f6_f32(manifest);
|
||||
initialize_block_scaled_gemm_reference_operations_fp4a_vs16(manifest);
|
||||
initialize_block_scaled_gemm_reference_operations_fp4a_vs32(manifest);
|
||||
initialize_block_scaled_gemm_reference_operations_mixed8bitsa(manifest);
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -211,6 +211,10 @@ protected:
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<typename Operator::GemmKernel::TileSchedulerTag, cutlass::gemm::StreamKScheduler>) {
|
||||
operator_args.scheduler.splits = arguments->split_k_slices;
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
|
||||
@@ -334,6 +334,7 @@ static struct {
|
||||
OperationKind_enumerants[] = {
|
||||
{"eq_gemm", "EqGemm", OperationKind::kEqGemm},
|
||||
{"gemm", "Gemm", OperationKind::kGemm},
|
||||
{"block_scaled_gemm", "blockScaledGemm", OperationKind::kBlockScaledGemm},
|
||||
{"rank_k", "RankK", OperationKind::kRankK},
|
||||
{"rank_2k", "Rank2K", OperationKind::kRank2K},
|
||||
{"trmm", "Trmm", OperationKind::kTrmm},
|
||||
@@ -422,6 +423,53 @@ Status from_string<Status>(std::string const &str) {
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
static struct {
|
||||
char const *text;
|
||||
char const *pretty;
|
||||
RuntimeDatatype enumerant;
|
||||
}
|
||||
RuntimeDatatype_enumerants[] = {
|
||||
{"e4m3", "<e4m3>", RuntimeDatatype::kE4M3},
|
||||
{"e5m2", "<e5m2>", RuntimeDatatype::kE5M2},
|
||||
{"e3m2", "<e3m2>", RuntimeDatatype::kE3M2},
|
||||
{"e2m3", "<e2m3>", RuntimeDatatype::kE2M3},
|
||||
{"e2m1", "<e2m1>", RuntimeDatatype::kE2M1}
|
||||
};
|
||||
|
||||
/// Converts a RuntimeDatatype enumerant to a string
|
||||
char const *to_string(RuntimeDatatype type, bool pretty) {
|
||||
|
||||
for (auto const & possible : RuntimeDatatype_enumerants) {
|
||||
if (type == possible.enumerant) {
|
||||
if (pretty) {
|
||||
return possible.pretty;
|
||||
}
|
||||
else {
|
||||
return possible.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pretty ? "Invalid" : "invalid";
|
||||
}
|
||||
|
||||
|
||||
/// Converts a RuntimeDatatype enumerant from a string
|
||||
template <>
|
||||
RuntimeDatatype from_string<RuntimeDatatype>(std::string const &str) {
|
||||
|
||||
for (auto const & possible : RuntimeDatatype_enumerants) {
|
||||
if ((str.compare(possible.text) == 0) ||
|
||||
(str.compare(possible.pretty) == 0)) {
|
||||
return possible.enumerant;
|
||||
}
|
||||
}
|
||||
|
||||
return RuntimeDatatype::kInvalid;
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static struct {
|
||||
@@ -447,6 +495,16 @@ NumericTypeID_enumerants[] = {
|
||||
{"s64", "S64", NumericTypeID::kS64},
|
||||
{"fe4m3", "FE4M3", NumericTypeID::kFE4M3},
|
||||
{"fe5m2", "FE5M2", NumericTypeID::kFE5M2},
|
||||
|
||||
{"f8", "F8", NumericTypeID::kF8},
|
||||
{"f6", "F6", NumericTypeID::kF6},
|
||||
{"f4", "F4", NumericTypeID::kF4},
|
||||
{"fe2m3", "FE2M3", NumericTypeID::kFE2M3},
|
||||
{"fe3m2", "FE3M2", NumericTypeID::kFE3M2},
|
||||
{"fe2m1", "FE2M1", NumericTypeID::kFE2M1},
|
||||
{"fue8m0", "FUE8M0", NumericTypeID::kFUE8M0},
|
||||
{"fue4m3", "FUE4M3", NumericTypeID::kFUE4M3},
|
||||
|
||||
{"f16", "F16", NumericTypeID::kF16},
|
||||
{"bf16", "BF16", NumericTypeID::kBF16},
|
||||
{"f32", "F32", NumericTypeID::kF32},
|
||||
@@ -510,6 +568,16 @@ int sizeof_bits(NumericTypeID type) {
|
||||
switch (type) {
|
||||
case NumericTypeID::kFE4M3: return 8;
|
||||
case NumericTypeID::kFE5M2: return 8;
|
||||
|
||||
case NumericTypeID::kF8: return 8;
|
||||
case NumericTypeID::kF6: return 6;
|
||||
case NumericTypeID::kF4: return 4;
|
||||
case NumericTypeID::kFE2M3: return 6;
|
||||
case NumericTypeID::kFE3M2: return 6;
|
||||
case NumericTypeID::kFE2M1: return 4;
|
||||
case NumericTypeID::kFUE8M0: return 8;
|
||||
case NumericTypeID::kFUE4M3: return 8;
|
||||
|
||||
case NumericTypeID::kF16: return 16;
|
||||
case NumericTypeID::kBF16: return 16;
|
||||
case NumericTypeID::kTF32: return 32;
|
||||
@@ -589,6 +657,16 @@ bool is_signed_type(NumericTypeID type) {
|
||||
switch (type) {
|
||||
case NumericTypeID::kFE4M3: return true;
|
||||
case NumericTypeID::kFE5M2: return true;
|
||||
|
||||
case NumericTypeID::kF8: return true;
|
||||
case NumericTypeID::kF6: return true;
|
||||
case NumericTypeID::kF4: return true;
|
||||
case NumericTypeID::kFE2M3: return true;
|
||||
case NumericTypeID::kFE3M2: return true;
|
||||
case NumericTypeID::kFE2M1: return true;
|
||||
case NumericTypeID::kFUE8M0: return false;
|
||||
case NumericTypeID::kFUE4M3: return false;
|
||||
|
||||
case NumericTypeID::kF16: return true;
|
||||
case NumericTypeID::kBF16: return true;
|
||||
case NumericTypeID::kTF32: return true;
|
||||
@@ -620,6 +698,16 @@ bool is_float_type(NumericTypeID type) {
|
||||
switch (type) {
|
||||
case NumericTypeID::kFE4M3: return true;
|
||||
case NumericTypeID::kFE5M2: return true;
|
||||
|
||||
case NumericTypeID::kF8: return true;
|
||||
case NumericTypeID::kF6: return true;
|
||||
case NumericTypeID::kF4: return true;
|
||||
case NumericTypeID::kFE2M3: return true;
|
||||
case NumericTypeID::kFE3M2: return true;
|
||||
case NumericTypeID::kFE2M1: return true;
|
||||
case NumericTypeID::kFUE8M0: return true;
|
||||
case NumericTypeID::kFUE4M3: return true;
|
||||
|
||||
case NumericTypeID::kF16: return true;
|
||||
case NumericTypeID::kBF16: return true;
|
||||
case NumericTypeID::kTF32: return true;
|
||||
@@ -1168,6 +1256,43 @@ bool lexical_cast(std::vector<uint8_t> &bytes, NumericTypeID type, std::string c
|
||||
*reinterpret_cast<float_e5m2_t *>(bytes.data()) = static_cast<float_e5m2_t>(tmp);
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kFE2M3:
|
||||
{
|
||||
float tmp;
|
||||
ss >> tmp;
|
||||
*reinterpret_cast<float_e2m3_t *>(bytes.data()) = static_cast<float_e2m3_t>(tmp);
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE3M2:
|
||||
{
|
||||
float tmp;
|
||||
ss >> tmp;
|
||||
*reinterpret_cast<float_e3m2_t *>(bytes.data()) = static_cast<float_e3m2_t>(tmp);
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE2M1:
|
||||
{
|
||||
float tmp;
|
||||
ss >> tmp;
|
||||
*reinterpret_cast<float_e2m1_t *>(bytes.data()) = static_cast<float_e2m1_t>(tmp);
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE8M0:
|
||||
{
|
||||
float tmp;
|
||||
ss >> tmp;
|
||||
*reinterpret_cast<float_ue8m0_t *>(bytes.data()) = static_cast<float_ue8m0_t>(tmp);
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE4M3:
|
||||
{
|
||||
float tmp;
|
||||
ss >> tmp;
|
||||
*reinterpret_cast<float_ue4m3_t *>(bytes.data()) = static_cast<float_ue4m3_t>(tmp);
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kF16:
|
||||
{
|
||||
float tmp;
|
||||
@@ -1317,6 +1442,38 @@ std::string lexical_cast(std::vector<uint8_t> &bytes, NumericTypeID type) {
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kFE2M3:
|
||||
{
|
||||
float tmp = *reinterpret_cast<float_e2m3_t *>(bytes.data());
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE3M2:
|
||||
{
|
||||
float tmp = *reinterpret_cast<float_e3m2_t *>(bytes.data());
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE2M1:
|
||||
{
|
||||
float tmp = *reinterpret_cast<float_e2m1_t *>(bytes.data());
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE8M0:
|
||||
{
|
||||
float tmp = *reinterpret_cast<float_ue8m0_t *>(bytes.data());
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE4M3:
|
||||
{
|
||||
float tmp = *reinterpret_cast<float_ue4m3_t *>(bytes.data());
|
||||
ss << tmp;
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kF16:
|
||||
{
|
||||
float tmp = *reinterpret_cast<half_t *>(bytes.data());
|
||||
@@ -1469,6 +1626,33 @@ bool cast_from_int64(std::vector<uint8_t> &bytes, NumericTypeID type, int64_t sr
|
||||
*reinterpret_cast<float_e5m2_t *>(bytes.data()) = static_cast<float_e5m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kFE2M3:
|
||||
{
|
||||
*reinterpret_cast<float_e2m3_t *>(bytes.data()) = static_cast<float_e2m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE3M2:
|
||||
{
|
||||
*reinterpret_cast<float_e3m2_t *>(bytes.data()) = static_cast<float_e3m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE2M1:
|
||||
{
|
||||
*reinterpret_cast<float_e2m1_t *>(bytes.data()) = static_cast<float_e2m1_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE8M0:
|
||||
{
|
||||
*reinterpret_cast<float_ue8m0_t *>(bytes.data()) = static_cast<float_ue8m0_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE4M3:
|
||||
{
|
||||
*reinterpret_cast<float_ue4m3_t *>(bytes.data()) = static_cast<float_ue4m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kF16:
|
||||
{
|
||||
*reinterpret_cast<half_t *>(bytes.data()) = static_cast<half_t>(float(src));
|
||||
@@ -1579,6 +1763,33 @@ bool cast_from_uint64(std::vector<uint8_t> &bytes, NumericTypeID type, uint64_t
|
||||
*reinterpret_cast<float_e5m2_t *>(bytes.data()) = static_cast<float_e5m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kFE2M3:
|
||||
{
|
||||
*reinterpret_cast<float_e2m3_t *>(bytes.data()) = static_cast<float_e2m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE3M2:
|
||||
{
|
||||
*reinterpret_cast<float_e3m2_t *>(bytes.data()) = static_cast<float_e3m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE2M1:
|
||||
{
|
||||
*reinterpret_cast<float_e2m1_t *>(bytes.data()) = static_cast<float_e2m1_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE8M0:
|
||||
{
|
||||
*reinterpret_cast<float_ue8m0_t *>(bytes.data()) = static_cast<float_ue8m0_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE4M3:
|
||||
{
|
||||
*reinterpret_cast<float_ue4m3_t *>(bytes.data()) = static_cast<float_ue4m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kF16:
|
||||
{
|
||||
*reinterpret_cast<half_t *>(bytes.data()) = static_cast<half_t>(float(src));
|
||||
@@ -1690,6 +1901,33 @@ bool cast_from_double(std::vector<uint8_t> &bytes, NumericTypeID type, double sr
|
||||
*reinterpret_cast<float_e5m2_t *>(bytes.data()) = static_cast<float_e5m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kFE2M3:
|
||||
{
|
||||
*reinterpret_cast<float_e2m3_t *>(bytes.data()) = static_cast<float_e2m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE3M2:
|
||||
{
|
||||
*reinterpret_cast<float_e3m2_t *>(bytes.data()) = static_cast<float_e3m2_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFE2M1:
|
||||
{
|
||||
*reinterpret_cast<float_e2m1_t *>(bytes.data()) = static_cast<float_e2m1_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE8M0:
|
||||
{
|
||||
*reinterpret_cast<float_ue8m0_t *>(bytes.data()) = static_cast<float_ue8m0_t>(float(src));
|
||||
}
|
||||
break;
|
||||
case NumericTypeID::kFUE4M3:
|
||||
{
|
||||
*reinterpret_cast<float_ue4m3_t *>(bytes.data()) = static_cast<float_ue4m3_t>(float(src));
|
||||
}
|
||||
break;
|
||||
|
||||
case NumericTypeID::kF16:
|
||||
{
|
||||
*reinterpret_cast<half_t *>(bytes.data()) = static_cast<half_t>(float(src));
|
||||
@@ -1751,6 +1989,35 @@ bool cast_from_double(std::vector<uint8_t> &bytes, NumericTypeID type, double sr
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
NumericTypeID dynamic_datatype_to_id(RuntimeDatatype type) {
|
||||
NumericTypeID element{};
|
||||
switch (type) {
|
||||
case RuntimeDatatype::kE4M3:
|
||||
element = NumericTypeID::kFE4M3;
|
||||
break;
|
||||
case RuntimeDatatype::kE5M2:
|
||||
element = NumericTypeID::kFE5M2;
|
||||
break;
|
||||
|
||||
case RuntimeDatatype::kE2M3:
|
||||
element = NumericTypeID::kFE2M3;
|
||||
break;
|
||||
case RuntimeDatatype::kE3M2:
|
||||
element = NumericTypeID::kFE3M2;
|
||||
break;
|
||||
case RuntimeDatatype::kE2M1:
|
||||
element = NumericTypeID::kFE2M1;
|
||||
break;
|
||||
|
||||
default:
|
||||
assert("illegal runtime datatype!");
|
||||
break;
|
||||
}
|
||||
return element;
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace library
|
||||
|
||||
Reference in New Issue
Block a user