[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes (#304)
* Merge with private repo * Update README * Update README * Update README * Add PyTorch requirements * Fix sync scopes for MQA logits (#256) * Update README
This commit is contained in:
+46
-23
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "../jit_kernels/heuristics/runtime.hpp"
|
||||
#include "../utils/layout.hpp"
|
||||
#include "../utils/compatibility.hpp"
|
||||
|
||||
@@ -12,21 +13,24 @@ namespace deep_gemm::layout {
|
||||
#if DG_TENSORMAP_COMPATIBLE
|
||||
static torch::Tensor transform_sf_into_required_layout(const torch::Tensor& sf,
|
||||
const int& mn, const int& k,
|
||||
const std::optional<std::tuple<int, int, int>>& recipe,
|
||||
const std::optional<std::tuple<int, int>>& recipe_ab,
|
||||
const std::variant<std::tuple<int, int, int>,
|
||||
std::tuple<int, int>>& recipe,
|
||||
const std::optional<int>& num_groups,
|
||||
const bool& is_sfa,
|
||||
const std::optional<bool>& is_sfa,
|
||||
const bool& disable_ue8m0_cast) {
|
||||
const auto& arch_major = device_runtime->get_arch_major();
|
||||
const auto arch_major = device_runtime->get_arch_major();
|
||||
|
||||
// Get granularity MN/K from recipe
|
||||
int gran_mn, gran_k;
|
||||
if (recipe.has_value()) {
|
||||
DG_HOST_ASSERT(not recipe_ab.has_value());
|
||||
gran_mn = is_sfa ? std::get<0>(recipe.value()) : std::get<1>(recipe.value());
|
||||
gran_k = std::get<2>(recipe.value());
|
||||
if (auto p = std::get_if<std::tuple<int, int, int>>(&recipe)) {
|
||||
DG_HOST_ASSERT(is_sfa.has_value());
|
||||
gran_mn = is_sfa.value() ? std::get<0>(*p) : std::get<1>(*p);
|
||||
gran_k = std::get<2>(*p);
|
||||
} else if (auto p = std::get_if<std::tuple<int, int>>(&recipe)) {
|
||||
DG_HOST_ASSERT(not is_sfa.has_value());
|
||||
std::tie(gran_mn, gran_k) = *p;
|
||||
} else {
|
||||
DG_HOST_ASSERT(recipe_ab.has_value());
|
||||
std::tie(gran_mn, gran_k) = recipe_ab.value();
|
||||
DG_HOST_UNREACHABLE("Invalid recipe");
|
||||
}
|
||||
|
||||
// Pre-transform checks
|
||||
@@ -43,8 +47,8 @@ static torch::Tensor transform_sf_into_required_layout(const torch::Tensor& sf,
|
||||
// (FP32, x, gran_k) on SM100: transform to (INT, 1, gran_k), TMA-aligned and MN-major
|
||||
if (sf.scalar_type() == torch::kFloat and (gran_k == 32 or gran_k == 128) and arch_major == 10) {
|
||||
DG_HOST_ASSERT(not disable_ue8m0_cast);
|
||||
const auto& broadcasted = gran_mn == 1 ? sf :
|
||||
sf.index_select(-2, torch::arange(mn, at::TensorOptions().device(sf.device())).floor_divide_(gran_mn));
|
||||
const auto broadcasted = gran_mn == 1 ? sf :
|
||||
sf.index_select(-2, torch::arange(mn, at::TensorOptions().device(sf.device())).floor_divide_(gran_mn));
|
||||
return get_mn_major_tma_aligned_packed_ue8m0_tensor(broadcasted);
|
||||
}
|
||||
|
||||
@@ -64,11 +68,19 @@ static std::tuple<torch::Tensor, torch::Tensor, int, int> transform_sf_pair_into
|
||||
const std::optional<int>& num_groups_a,
|
||||
const std::optional<int>& num_groups_b,
|
||||
const bool& disable_ue8m0_cast = false) {
|
||||
DG_HOST_ASSERT(recipe_a.has_value() == recipe_b.has_value());
|
||||
// Use default recipe, if none is specified
|
||||
if (not recipe_a.has_value() and not recipe.has_value())
|
||||
recipe = get_default_recipe(sfa.scalar_type(), sfb.scalar_type());
|
||||
const auto transformed_sfa = transform_sf_into_required_layout(sfa, m, k, recipe, recipe_a, num_groups_a, true, disable_ue8m0_cast);
|
||||
const auto transformed_sfb = transform_sf_into_required_layout(sfb, n, k, recipe, recipe_b, num_groups_b, false, disable_ue8m0_cast);
|
||||
|
||||
// Must be either 'recipe' or the 'recipe_a' + 'recipe_b' pair.
|
||||
DG_HOST_ASSERT(recipe_a.has_value() == recipe_b.has_value());
|
||||
DG_HOST_ASSERT(recipe_a.has_value() != recipe.has_value());
|
||||
|
||||
// Transform SFA and SFB layout
|
||||
const auto transformed_sfa = recipe.has_value() ? transform_sf_into_required_layout(sfa, m, k, recipe.value(), num_groups_a, true, disable_ue8m0_cast)
|
||||
: transform_sf_into_required_layout(sfa, m, k, recipe_a.value(), num_groups_a, std::nullopt, disable_ue8m0_cast);
|
||||
const auto transformed_sfb = recipe.has_value() ? transform_sf_into_required_layout(sfb, n, k, recipe.value(), num_groups_b, false, disable_ue8m0_cast)
|
||||
: transform_sf_into_required_layout(sfb, n, k, recipe_b.value(), num_groups_b, std::nullopt, disable_ue8m0_cast);
|
||||
const int gran_k_a = recipe_a.has_value() ? std::get<1>(recipe_a.value()) : std::get<2>(recipe.value());
|
||||
const int gran_k_b = recipe_b.has_value() ? std::get<1>(recipe_b.value()) : std::get<2>(recipe.value());
|
||||
return std::make_tuple(transformed_sfa, transformed_sfb, gran_k_a, gran_k_b);
|
||||
@@ -79,8 +91,12 @@ static torch::Tensor transform_k_grouped_sf_into_required_layout(const torch::Te
|
||||
const torch::Tensor& ks_tensor,
|
||||
const std::tuple<int, int, int>& recipe) {
|
||||
DG_HOST_ASSERT(sf.dim() == 2);
|
||||
DG_HOST_ASSERT(recipe == std::make_tuple(1, 1, 128));
|
||||
const auto& arch_major = device_runtime->get_arch_major();
|
||||
DG_HOST_ASSERT(std::get<0>(recipe) == 1 and std::get<1>(recipe) == 1);
|
||||
|
||||
const int gran_k = std::get<2>(recipe);
|
||||
DG_HOST_ASSERT(gran_k == 32 or gran_k == 128);
|
||||
|
||||
const auto arch_major = device_runtime->get_arch_major();
|
||||
|
||||
// FP32 on SM90
|
||||
if (sf.scalar_type() == torch::kFloat and arch_major == 9)
|
||||
@@ -88,7 +104,7 @@ static torch::Tensor transform_k_grouped_sf_into_required_layout(const torch::Te
|
||||
|
||||
// FP32 on SM100
|
||||
if (sf.scalar_type() == torch::kFloat and arch_major == 10)
|
||||
return get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(sf, ks_tensor, ks);
|
||||
return get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor(sf, ks_tensor, ks, gran_k);
|
||||
|
||||
// INT on SM100
|
||||
if (sf.scalar_type() == torch::kInt and arch_major == 10)
|
||||
@@ -100,12 +116,11 @@ static torch::Tensor transform_k_grouped_sf_into_required_layout(const torch::Te
|
||||
#endif
|
||||
|
||||
static void register_apis(pybind11::module_& m) {
|
||||
|
||||
#if DG_TENSORMAP_COMPATIBLE
|
||||
m.def("transform_sf_into_required_layout", &transform_sf_into_required_layout,
|
||||
py::arg("sf"), py::arg("mn"), py::arg("k"),
|
||||
py::arg("recipe") = std::nullopt, py::arg("recipe_ab") = std::nullopt,
|
||||
py::arg("num_groups") = std::nullopt, py::arg("is_sfa") = false,
|
||||
py::arg("sf"), py::arg("mn"), py::arg("k"), py::arg("recipe"),
|
||||
py::arg("num_groups") = std::nullopt,
|
||||
py::arg("is_sfa") = std::nullopt,
|
||||
py::arg("disable_ue8m0_cast") = false);
|
||||
|
||||
m.def("get_tma_aligned_size", &get_tma_aligned_size);
|
||||
@@ -114,7 +129,15 @@ static void register_apis(pybind11::module_& m) {
|
||||
m.def("get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor", &get_k_grouped_mn_major_tma_aligned_packed_ue8m0_tensor);
|
||||
#endif
|
||||
|
||||
m.def("get_mk_alignment_for_contiguous_layout", &get_mk_alignment_for_contiguous_layout);
|
||||
m.def("set_mk_alignment_for_contiguous_layout", [&](const int& new_value) {
|
||||
heuristics_runtime->set_mk_alignment_for_contiguous_layout(new_value);
|
||||
});
|
||||
m.def("get_mk_alignment_for_contiguous_layout", [&]() {
|
||||
return heuristics_runtime->get_mk_alignment_for_contiguous_layout();
|
||||
});
|
||||
m.def("get_theoretical_mk_alignment_for_contiguous_layout", [&](const std::optional<int>& expected_m) {
|
||||
return heuristics_runtime->get_theoretical_mk_alignment_for_contiguous_layout(expected_m);
|
||||
}, py::arg("expected_m") = std::nullopt);
|
||||
}
|
||||
|
||||
} // namespace deep_gemm::layout
|
||||
|
||||
Reference in New Issue
Block a user