/*************************************************************************************************** * Copyright (c) 2025 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: BSD-3-Clause * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * **************************************************************************************************/ /*! \file \brief An FP8 blockwise scaled GEMM example for the NVIDIA Blackwell SM100 architecture using CUTLASS. */ #include #include "cutlass/cutlass.h" #include "cute/tensor.hpp" #include "cutlass/tensor_ref.h" #include "cutlass/epilogue/thread/activation.h" #include "cutlass/gemm/dispatch_policy.hpp" #include "cutlass/gemm/collective/collective_builder.hpp" #include "cutlass/epilogue/dispatch_policy.hpp" #include "cutlass/epilogue/collective/collective_builder.hpp" #include "cutlass/gemm/device/gemm_universal_adapter.h" #include "cutlass/gemm/kernel/gemm_universal.hpp" #include "cutlass/gemm/kernel/tile_scheduler_params.h" #include "cutlass/util/command_line.h" #include "cutlass/util/distribution.h" #include "cutlass/util/host_tensor.h" #include "cutlass/util/packed_stride.hpp" #include "cutlass/util/tensor_view_io.h" #include "cutlass/util/reference/host/tensor_fill.h" #include "cutlass/util/reference/host/tensor_copy.h" #include "cutlass/util/reference/host/tensor_compare.h" #include "cutlass/util/reference/host/tensor_norm.h" #include "cutlass/util/reference/host/gett.hpp" #include "helper.h" using namespace cute; #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) ///////////////////////////////////////////////////////////////////////////////////////////////// /// GEMM kernel configurations ///////////////////////////////////////////////////////////////////////////////////////////////// // A matrix configuration using ElementA = cutlass::float_e4m3_t; // Element type for A matrix operand using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes) // B matrix configuration using ElementB = cutlass::float_e4m3_t; // Element type for B matrix operand using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes) // C/D matrix configuration using ElementC = cutlass::float_e4m3_t; // Element type for C and D matrix operands using LayoutC = cutlass::layout::ColumnMajor; // Layout type for C and D matrix operands constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A matrix in units of elements (up to 16 bytes) using ElementD = ElementC; using LayoutD = LayoutC; constexpr int AlignmentD = AlignmentC; // MMA type using ElementAccumulator = float; // Element Accumulator will also be our scale factor type using ElementCompute = float; // MMA and Cluster Tile Shapes // Shape of the tile computed by tcgen05 MMA, could be across 2 SMs if Cluster Shape %2 == 0 using MmaTileShape_MNK = Shape<_128,_128,_128>; // Shape of the threadblocks in a cluster using ClusterShape_MNK = Shape<_1,_1,_1>; using ScaleConfig = decltype(cutlass::detail::sm100_trivial_blockwise_scale_config(MmaTileShape_MNK{})); using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); // Layout type for SFA matrix operand using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); // Layout type for SFB matrix operand using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, MmaTileShape_MNK, ClusterShape_MNK, cutlass::epilogue::collective::EpilogueTileAuto, ElementAccumulator, ElementCompute, ElementC, LayoutC, AlignmentC, ElementD, LayoutC, AlignmentD, cutlass::epilogue::collective::EpilogueScheduleAuto >::CollectiveOp; using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, ElementA, cute::tuple, AlignmentA, ElementB, cute::tuple, AlignmentB, ElementAccumulator, MmaTileShape_MNK, ClusterShape_MNK, cutlass::gemm::collective::StageCountAutoCarveout(sizeof(typename CollectiveEpilogue::SharedStorage))>, cutlass::gemm::KernelScheduleSm100Blockwise >::CollectiveOp; using GemmKernel = cutlass::gemm::kernel::GemmUniversal< Shape, CollectiveMainloop, CollectiveEpilogue, void>; // Default to ClusterLaunchControl (CLC) based tile scheduler using Gemm = cutlass::gemm::device::GemmUniversalAdapter; using StrideA = typename Gemm::GemmKernel::StrideA; using StrideB = typename Gemm::GemmKernel::StrideB; using StrideC = typename Gemm::GemmKernel::StrideC; using StrideD = typename Gemm::GemmKernel::StrideD; /// Initialization StrideA stride_A; StrideB stride_B; StrideC stride_C; StrideD stride_D; // Strides just iterate over scalars and have no zeros LayoutSFA layout_SFA; LayoutSFB layout_SFB; // Layouts are tiled to the problem size and the strides have zeros uint64_t seed; cutlass::HostTensor tensor_A; cutlass::HostTensor tensor_SFA; cutlass::HostTensor tensor_B; cutlass::HostTensor tensor_SFB; cutlass::HostTensor tensor_C; cutlass::HostTensor tensor_D; cutlass::HostTensor tensor_ref_D; #endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) ///////////////////////////////////////////////////////////////////////////////////////////////// /// Testbed utility types ///////////////////////////////////////////////////////////////////////////////////////////////// // Command line options parsing struct Options { bool help = false; bool skip_verification = false; float alpha = 1.f, beta = 0.f; int iterations = 1000; int m = 1024, n = 512, k = 1024, l = 1; // Parses the command line void parse(int argc, char const **args) { cutlass::CommandLine cmd(argc, args); if (cmd.check_cmd_line_flag("help")) { help = true; return; } if (cmd.check_cmd_line_flag("skip-verification")) { skip_verification = true; } cmd.get_cmd_line_argument("m", m); cmd.get_cmd_line_argument("n", n); cmd.get_cmd_line_argument("k", k); cmd.get_cmd_line_argument("l", l); cmd.get_cmd_line_argument("alpha", alpha, 1.f); cmd.get_cmd_line_argument("beta", beta, 0.f); cmd.get_cmd_line_argument("iterations", iterations); } /// Prints the usage statement. std::ostream & print_usage(std::ostream &out) const { out << "81_blackwell_gemm_blockwise\n\n" << " Blackwell FP8 GEMM with Blockwise Scaling using a Warp Specialized kernel.\n\n" << "Options:\n\n" << " --help If specified, displays this usage statement\n\n" << " --m= Sets the M extent of the GEMM\n" << " --n= Sets the N extent of the GEMM\n" << " --k= Sets the K extent of the GEMM\n" << " --l= Sets the l extent (batch) of the GEMM\n" << " --alpha= Epilogue scalar alpha\n" << " --beta= Epilogue scalar beta\n" << " --iterations= Number of profiling iterations to perform.\n\n" << " --skip-verification Skip verification.\n\n"; out << "\n\nExamples:\n\n" << "$ " << "112_blackwell_gemm_blockwise" << " --m=1024 --n=512 --k=1024 --alpha=2 --beta=0.707 \n\n"; return out; } /// Compute performance in GFLOP/s double gflops(double runtime_s) const { // Two flops per multiply-add uint64_t flop = uint64_t(2) * m * n * k; double gflop = double(flop) / double(1.0e9); return gflop / runtime_s; } }; /// Result structure struct Result { double avg_runtime_ms; double gflops; cutlass::Status status; cudaError_t error; bool passed; Result( double avg_runtime_ms = 0, double gflops = 0, cutlass::Status status = cutlass::Status::kSuccess, cudaError_t error = cudaSuccess) : avg_runtime_ms(avg_runtime_ms), gflops(gflops), status(status), error(error), passed(false) {} }; #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) ///////////////////////////////////////////////////////////////////////////////////////////////// /// GEMM setup and evaluation ///////////////////////////////////////////////////////////////////////////////////////////////// /// Helper to initialize a block of device data template bool initialize_tensor( cutlass::TensorView view, cutlass::Distribution::Kind dist_kind, uint64_t seed) { if (dist_kind == cutlass::Distribution::Uniform) { double scope_max, scope_min; int bits_input = cutlass::sizeof_bits::value; int bits_output = cutlass::sizeof_bits::value; if (bits_input == 1) { scope_max = 2; scope_min = 0; } else if (bits_input <= 8) { scope_max = 2; scope_min = -2; } else if (bits_output == 16) { scope_max = 5; scope_min = -5; } else { scope_max = 8; scope_min = -8; } cutlass::reference::host::TensorFillRandomUniform( view, seed, scope_max, scope_min, 0); } else if (dist_kind == cutlass::Distribution::AllZeros) { cutlass::reference::host::TensorFill(view); } else if (dist_kind == cutlass::Distribution::Identity) { cutlass::reference::host::TensorFillIdentity(view); } else if (dist_kind == cutlass::Distribution::Gaussian) { cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5); } else if (dist_kind == cutlass::Distribution::Sequential) { cutlass::reference::host::BlockFillSequential(view.data(), view.capacity()); } else { throw std::runtime_error("Not implementated."); } return true; } /// Helper to initialize a block of device data (scale_tensors) template bool initialize_scale_tensor( cutlass::TensorView view, cutlass::Distribution::Kind dist_kind, uint64_t seed) { if (dist_kind == cutlass::Distribution::Uniform) { double scope_max, scope_min; scope_min = -8; scope_max = 8; cutlass::reference::host::TensorFillRandomUniform( view, seed, scope_max, scope_min, 0); } else if (dist_kind == cutlass::Distribution::AllZeros) { cutlass::reference::host::TensorFill(view); } else if (dist_kind == cutlass::Distribution::Identity) { cutlass::reference::host::TensorFillIdentity(view); } else if (dist_kind == cutlass::Distribution::Gaussian) { cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5); } else if (dist_kind == cutlass::Distribution::Sequential) { cutlass::reference::host::BlockFillSequential(view.data(), view.capacity()); } else { throw std::runtime_error("Not implementated."); } return true; } /// Initialize operands to be used in the GEMM and reference GEMM void initialize(const Options &options) { using namespace cute; auto gemm_problem_shape = cute::make_shape(options.m, options.n, options.k); stride_A = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(options.m, options.k, options.l)); stride_B = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(options.n, options.k, options.l)); stride_C = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(options.m, options.n, options.l)); stride_D = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(options.m, options.n, options.l)); layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(options.m, options.n, options.k, options.l)); layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(options.m, options.n, options.k, options.l)); auto a_coord = cutlass::make_Coord(options.m * options.l, options.k); auto c_coord = cutlass::make_Coord(options.m * options.l, options.n); auto b_coord = cutlass::make_Coord(options.k, options.n * options.l); auto blockscale_a_coord = cutlass::make_Coord(size(filter_zeros(layout_SFA))); auto blockscale_b_coord = cutlass::make_Coord(size(filter_zeros(layout_SFB))); tensor_A.resize(a_coord); tensor_B.resize(b_coord); tensor_C.resize(c_coord); tensor_D.resize(c_coord); tensor_ref_D.resize(c_coord); tensor_SFA.resize(blockscale_a_coord); tensor_SFB.resize(blockscale_b_coord); initialize_tensor(tensor_A.host_view(), cutlass::Distribution::Uniform, seed + 2022); initialize_tensor(tensor_B.host_view(), cutlass::Distribution::Uniform, seed + 2023); initialize_tensor(tensor_C.host_view(), cutlass::Distribution::Uniform, seed + 2024); initialize_scale_tensor(tensor_SFA.host_view(), cutlass::Distribution::Uniform, seed + 2025); initialize_scale_tensor(tensor_SFB.host_view(), cutlass::Distribution::Uniform, seed + 2026); tensor_A.sync_device(); tensor_B.sync_device(); tensor_C.sync_device(); tensor_D.sync_device(); tensor_SFA.sync_device(); tensor_SFB.sync_device(); } /// Populates a Gemm::Arguments structure from the given commandline options typename Gemm::Arguments args_from_options(const Options &options) { typename Gemm::Arguments arguments{ cutlass::gemm::GemmUniversalMode::kGemm, {options.m, options.n, options.k, options.l}, {tensor_A.device_data(), stride_A, tensor_B.device_data(), stride_B, tensor_SFA.device_data(), layout_SFA, tensor_SFB.device_data(), layout_SFB}, { {}, // epilogue.thread tensor_C.device_data(), stride_C, tensor_D.device_data(), stride_D } }; auto &fusion_args = arguments.epilogue.thread; fusion_args.alpha = options.alpha; fusion_args.beta = options.beta; return arguments; } bool verify(const Options &options) { // // Compute reference output // // Create instantiation for device reference gemm kernel auto A = cute::make_tensor(tensor_A.host_data(), cute::make_layout(cute::make_shape(options.m, options.k, options.l), stride_A)); auto B = cute::make_tensor(tensor_B.host_data(), cute::make_layout(cute::make_shape(options.n, options.k, options.l), stride_B)); auto C = cute::make_tensor(tensor_C.host_data(), cute::make_layout(cute::make_shape(options.m, options.n, options.l), stride_C)); auto D = cute::make_tensor(tensor_ref_D.host_data(), cute::make_layout(cute::make_shape(options.m, options.n, options.l), stride_D)); auto SFA = cute::make_tensor(tensor_SFA.host_data(), layout_SFA); auto SFB = cute::make_tensor(tensor_SFB.host_data(), layout_SFB); using unused_t = decltype(D); cutlass::reference::host::GettBlockScalingMainloopParams< ElementAccumulator, decltype(A), decltype(SFA), decltype(B), decltype(SFB) > mainloop_params{A, SFA, B, SFB}; cutlass::reference::host::GettEpilogueParams< ElementAccumulator, ElementAccumulator, ElementAccumulator, ElementCompute, decltype(C), decltype(D) > epilogue_params; epilogue_params.C = C; epilogue_params.D = D; epilogue_params.alpha = options.alpha; epilogue_params.beta = options.beta; // get reference result cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params); // compare_reference tensor_D.sync_host(); bool passed = cutlass::reference::host::TensorEquals(tensor_ref_D.host_view(), tensor_D.host_view()); return passed; } /// Execute a given example GEMM computation template int run(Options &options) { initialize(options); // Instantiate CUTLASS kernel depending on templates Gemm gemm; // Create a structure of gemm kernel arguments suitable for invoking an instance of Gemm auto arguments = args_from_options(options); // Using the arguments, query for extra workspace required for matrix multiplication computation size_t workspace_size = Gemm::get_workspace_size(arguments); // Allocate workspace memory cutlass::device_memory::allocation workspace(workspace_size); // Check if the problem size is supported or not CUTLASS_CHECK(gemm.can_implement(arguments)); // Initialize CUTLASS kernel with arguments and workspace pointer CUTLASS_CHECK(gemm.initialize(arguments, workspace.get())); // Correctness / Warmup iteration CUTLASS_CHECK(gemm.run()); Result result; if (!options.skip_verification) { // Check if output from CUTLASS kernel and reference kernel are equal or not result.passed = verify(options); std::cout << " Disposition: " << (result.passed ? "Passed" : "Failed") << std::endl; if (!result.passed) { exit(-1); } } // Run profiling loop if (options.iterations > 0) { GpuTimer timer; timer.start(); for (int iter = 0; iter < options.iterations; ++iter) { CUTLASS_CHECK(gemm.run()); } timer.stop(); // Compute average runtime and GFLOPs. float elapsed_ms = timer.elapsed_millis(); result.avg_runtime_ms = double(elapsed_ms) / double(options.iterations); result.gflops = options.gflops(result.avg_runtime_ms / 1000.0); std::cout << " Problem Size: " << options.m << 'x' << options.n << 'x' << options.k << 'x' << options.l << std::endl; std::cout << " Avg runtime: " << result.avg_runtime_ms << " ms" << std::endl; std::cout << " GFLOPS: " << result.gflops << std::endl; } return 0; } #endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) /////////////////////////////////////////////////////////////////////////////////////////////////// int main(int argc, char const **args) { // CUTLASS must be compiled with CUDA 12.0 Toolkit to run this example // and must have compute capability at least sm100a. if (__CUDACC_VER_MAJOR__ < 12) { std::cerr << "This example requires CUDA 12 or newer.\n"; // Returning zero so this test passes on older Toolkits. Its actions are no-op. return 0; } cudaDeviceProp props; int current_device_id; CUDA_CHECK(cudaGetDevice(¤t_device_id)); CUDA_CHECK(cudaGetDeviceProperties(&props, current_device_id)); cudaError_t error = cudaGetDeviceProperties(&props, 0); if (props.major != 10 || props.minor != 0) { std::cerr << "This example requires a GPU with compute capability 100a)." << std::endl; return 0; } // // Parse options // Options options; options.parse(argc, args); if (options.help) { options.print_usage(std::cout) << std::endl; return 0; } // // Run // #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) run(options); #endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) return 0; } /////////////////////////////////////////////////////////////////////////////////////////////////