#pragma once #include #include "../../jit/compiler.hpp" #include "../../jit/kernel_runtime.hpp" #include "../../utils/exception.hpp" #include "../../utils/format.hpp" #include "runtime_utils.hpp" #include #include #include "../heuristics/sm90_mega_moe.hpp" namespace deep_gemm { class SM90FP8MegaMoERuntime final : public LaunchRuntime { public: struct Args { // Templated arguments int num_max_tokens_per_rank; int hidden, intermediate_hidden; int num_experts, num_topk; int num_ranks; float activation_clamp; bool fast_math; SM90MegaMoEConfig config; // Runtime arguments void* y; void* l1_accum_debug; int* cumulative_local_expert_recv_stats; int num_tokens; layout::SymBuffer<> sym_buffer_ptrs; // Tensormap CUtensorMap tensor_map_l1_acts; CUtensorMap tensor_map_l1_acts_sf; CUtensorMap tensor_map_l1_weights; CUtensorMap tensor_map_l1_output; CUtensorMap tensor_map_l2_acts; CUtensorMap tensor_map_l2_acts_sf; CUtensorMap tensor_map_l2_weights; void* l1_weights_sf; void* l2_weights_sf; // Launch configs LaunchArgs launch_args; }; static std::string generate_impl(const Args& args) { return fmt::format(R"( #include using namespace deep_gemm; static void __instantiate_kernel() {{ auto ptr = reinterpret_cast(&sm90_fp8_mega_moe_impl< {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} >); }}; )", args.num_max_tokens_per_rank, args.hidden, args.intermediate_hidden, args.num_experts, args.num_topk, args.config.num_experts_per_wave, args.config.block_m, args.config.block_n, args.config.block_k, args.config.store_block_m, args.config.num_max_pool_tokens, args.config.num_padded_sf_pool_tokens, args.config.num_stages, args.config.num_dispatch_threads, args.config.num_tma_threads, args.config.num_math_threads, args.config.cooperative ? "true" : "false", args.config.use_n_major_l2 ? "true" : "false", args.launch_args.grid_dim.first, args.num_ranks, to_string(args.activation_clamp), args.fast_math ? "true" : "false"); } static void launch_impl(const KernelHandle& kernel, const LaunchConfigHandle& config, Args args) { DG_CUDA_UNIFIED_CHECK(launch_kernel(kernel, config, args.y, args.cumulative_local_expert_recv_stats, args.num_tokens, args.sym_buffer_ptrs, args.tensor_map_l1_acts, args.tensor_map_l1_acts_sf, args.tensor_map_l1_weights, args.l1_weights_sf, args.tensor_map_l1_output, args.tensor_map_l2_acts, args.tensor_map_l2_acts_sf, args.tensor_map_l2_weights, args.l2_weights_sf, args.l1_accum_debug )); } }; static void sm90_fp8_mega_moe( const torch::Tensor& y, const torch::Tensor& l1_acts, const torch::Tensor& l1_acts_sf, const torch::Tensor& l2_acts, const torch::Tensor& l2_acts_sf, const torch::Tensor& l1_weights, const torch::Tensor& l2_weights, const torch::Tensor& l1_weights_sf, const torch::Tensor& l2_weights_sf, const torch::Tensor& l1_accum_debug, const std::optional cumulative_local_expert_recv_stats, const std::vector& sym_buffer_ptrs, const int& rank_idx, const int& num_max_tokens_per_rank, const int& num_experts_per_rank, const int& num_tokens, const int& num_topk, const int& hidden, const int& intermediate_hidden, const float& activation_clamp, const bool& fast_math ) { const auto num_ranks = static_cast(sym_buffer_ptrs.size()); const auto num_experts = num_experts_per_rank * num_ranks; const auto num_padded_sf_pool_tokens = static_cast(l1_acts_sf.size(0)); // Heuristics const auto config = get_sm90_mega_moe_config( num_ranks, num_experts, num_experts_per_rank, num_max_tokens_per_rank, num_tokens, num_topk, hidden, intermediate_hidden, num_padded_sf_pool_tokens); // Make tensormap constexpr int kGranK = 128; const auto tensor_map_l1_acts = make_tma_2d_desc(l1_acts, hidden, config.num_max_pool_tokens, config.block_k, config.block_m, static_cast(l1_acts.stride(-2)), 128); const auto tensor_map_l1_acts_sf = make_tma_sf_desc(cute::UMMA::Major::MN, l1_acts_sf, config.num_padded_sf_pool_tokens, hidden, config.block_m, kGranK, 1, 0); const auto tensor_map_l1_weights = make_tma_2d_desc(l1_weights, hidden, num_experts_per_rank * intermediate_hidden * 2, config.block_k, config.block_n, static_cast(l1_weights.stride(-2)), 128); // L1 output SwiGLU has half N width const auto tensor_map_l1_output = make_tma_2d_desc(l2_acts, intermediate_hidden, config.num_max_pool_tokens, config.block_n / 2, config.store_block_m, static_cast(l2_acts.stride(-2)), 64); const auto tensor_map_l2_acts = make_tma_2d_desc(l2_acts, intermediate_hidden, config.num_max_pool_tokens, config.block_k, config.block_m, static_cast(l2_acts.stride(-2)), 128); const auto tensor_map_l2_acts_sf = make_tma_sf_desc(cute::UMMA::Major::MN, l2_acts_sf, config.num_padded_sf_pool_tokens, intermediate_hidden, config.block_m, kGranK, 1, 0); const auto tensor_map_l2_weights = make_tma_2d_desc(l2_weights, intermediate_hidden, num_experts_per_rank * hidden, config.block_k, config.block_n, static_cast(l2_weights.stride(-2)), 128); // Stats can be optional int* cumulative_local_expert_recv_stats_ptr = nullptr; if (cumulative_local_expert_recv_stats.has_value()) cumulative_local_expert_recv_stats_ptr = cumulative_local_expert_recv_stats->data_ptr(); // Launch const auto num_sms = device_runtime->get_num_sms(); const int num_threads = config.num_dispatch_threads + config.num_tma_threads + config.num_math_threads; const SM90FP8MegaMoERuntime::Args args = { .num_max_tokens_per_rank = num_max_tokens_per_rank, .hidden = hidden, .intermediate_hidden = intermediate_hidden, .num_experts = num_experts, .num_topk = num_topk, .num_ranks = num_ranks, .activation_clamp = activation_clamp, .fast_math = fast_math, .config = config, .y = y.data_ptr(), .l1_accum_debug = l1_accum_debug.data_ptr(), .cumulative_local_expert_recv_stats = cumulative_local_expert_recv_stats_ptr, .num_tokens = num_tokens, .sym_buffer_ptrs = layout::SymBuffer<>(sym_buffer_ptrs, rank_idx), .tensor_map_l1_acts = tensor_map_l1_acts, .tensor_map_l1_acts_sf = tensor_map_l1_acts_sf, .tensor_map_l1_weights = tensor_map_l1_weights, .tensor_map_l1_output = tensor_map_l1_output, .tensor_map_l2_acts = tensor_map_l2_acts, .tensor_map_l2_acts_sf = tensor_map_l2_acts_sf, .tensor_map_l2_weights = tensor_map_l2_weights, .l1_weights_sf = l1_weights_sf.data_ptr(), .l2_weights_sf = l2_weights_sf.data_ptr(), .launch_args = LaunchArgs(num_sms, num_threads, config.smem_size) }; const auto code = SM90FP8MegaMoERuntime::generate(args); const auto runtime = compiler->build("sm90_fp8_mega_moe", code); SM90FP8MegaMoERuntime::launch(runtime, args); } } // namespace deep_gemm