Multiple updates and refactorings (#231)

This commit is contained in:
Ray Wang
2025-11-21 17:49:47 +08:00
committed by GitHub
parent bb4424aad4
commit 38f8ef73a4
80 changed files with 3767 additions and 2103 deletions
+5
View File
@@ -0,0 +1,5 @@
# All kernels may be deprecated in the future (or rewrite in TileLang)
from .m_grouped_gemm import *
from .a_fused_m_grouped_gemm import *
from .a_fused_k_grouped_gemm import *
from .b_fused_k_grouped_gemm import *
@@ -0,0 +1,88 @@
import torch
import triton
import triton.language as tl
from typing import Tuple
from .tune_options import *
from .._C import get_mk_alignment_for_contiguous_layout
@triton.autotune(configs=get_k_grouped_gemm_configs(), key=[], restore_value=['d_ptr'])
@triton.jit
def a_fused_k_grouped_bf16_gemm_contiguous_tl_impl(a_ptr, b_ptr, d_ptr,
k_indices_ptr, k_start_ptr, k_end_ptr,
M: tl.constexpr,
N: tl.constexpr,
K,
ACC: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr):
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
pid_b = (pid // (num_pid_m * num_pid_n)).to(tl.int64)
pid = pid % (num_pid_m * num_pid_n)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + (pid % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
m_range = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
n_range = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
m_range = tl.max_contiguous(tl.multiple_of(m_range, BLOCK_SIZE_M), BLOCK_SIZE_M)
n_range = tl.max_contiguous(tl.multiple_of(n_range, BLOCK_SIZE_N), BLOCK_SIZE_N)
m_mask = (m_range < M)[:, None]
n_mask = (n_range < N)[None, :]
k_start = tl.load(k_start_ptr + pid_b)
k_end = tl.load(k_end_ptr + pid_b)
if k_start >= k_end:
if not ACC:
d_ptrs = d_ptr + pid_b * M * N + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=d_ptr.dtype.element_ty), mask=m_mask & n_mask)
return
# Compute
acc = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for k in range(k_start, k_end, BLOCK_SIZE_K):
k_range = k + tl.arange(0, BLOCK_SIZE_K)
rows = tl.load(k_indices_ptr + k_range).to(tl.int64)
a_ptrs = a_ptr + m_range[:, None] + rows[None, :] * M
b_ptrs = b_ptr + k_range[:, None].to(tl.int64) * N + n_range[None, :]
a = tl.load(a_ptrs, mask=(rows >= 0)[None, :] & m_mask, other=0)
b = tl.load(b_ptrs, mask=n_mask, other=0)
acc = tl.dot(a, b, acc)
# Write back
d_ptrs = d_ptr + pid_b * M * N + m_range[:, None].to(tl.int64) * N + n_range[None, :]
if ACC:
acc += tl.load(d_ptrs, mask=m_mask & n_mask)
acc = acc.to(d_ptr.dtype.element_ty)
tl.store(d_ptrs, acc, mask=m_mask & n_mask)
def a_fused_k_grouped_bf16_gemm_tn_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
handle: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], acc: bool):
k_indices, k_start, k_end = handle
assert a.is_contiguous() and b.is_contiguous() and d.is_contiguous()
assert k_indices.is_contiguous() and k_start.is_contiguous() and k_end.is_contiguous()
assert a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
assert k_indices.dtype == torch.int32 and k_start.dtype == torch.int32 and k_end.dtype == torch.int32
assert a.dim() == 2 and b.dim() == 2 and d.dim() == 3
assert k_start.numel() == k_end.numel() and k_indices.size(0) == b.size(0)
assert d.size(0) == k_start.numel() and d.size(1) == a.size(1) and d.size(2) == b.size(1)
assert b.size(0) % get_mk_alignment_for_contiguous_layout() == 0
K_, M = a.shape
K, N = b.shape
B = k_start.numel()
grid = lambda META: (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N']) * B,)
a_fused_k_grouped_bf16_gemm_contiguous_tl_impl[grid](
a, b, d, k_indices, k_start, k_end, M, N, K, ACC=acc)
@@ -0,0 +1,92 @@
import torch
import triton
import triton.language as tl
from typing import Tuple
from .tune_options import *
from .._C import get_mk_alignment_for_contiguous_layout
@triton.autotune(configs=get_m_grouped_gemm_configs(), key=[])
@triton.jit
def a_fused_m_grouped_bf16_gemm_contiguous_tl_impl(a_ptr, b_ptr, d_ptr,
m_indices_ptr, m_row_indices_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr,
IS_B_K_MAJOR: tl.constexpr):
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + (pid % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
m_range = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
n_range = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
m_range = tl.max_contiguous(tl.multiple_of(m_range, BLOCK_SIZE_M), BLOCK_SIZE_M)
n_range = tl.max_contiguous(tl.multiple_of(n_range, BLOCK_SIZE_N), BLOCK_SIZE_N)
n_mask = (n_range < N)[None, :]
batch_id = tl.load(m_indices_ptr + pid_m * BLOCK_SIZE_M).to(tl.int64)
if batch_id < 0:
d_ptrs = d_ptr + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=d_ptr.dtype.element_ty), mask=n_mask)
return
# b block
rows = tl.load(m_row_indices_ptr + m_range).to(tl.int64)
# Compute
acc = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for k in range(0, K, BLOCK_SIZE_K):
k_range = (k + tl.arange(0, BLOCK_SIZE_K)).to(tl.int64)
k_mask = k_range < K
a_ptrs = a_ptr + rows[:, None] * K + k_range[None, :]
b_ptrs = b_ptr + batch_id * K * N + k_range[:, None] * (1 if IS_B_K_MAJOR else N) + n_range[None, :].to(tl.int64) * (K if IS_B_K_MAJOR else 1)
a = tl.load(a_ptrs, mask=(rows >= 0)[:, None] & k_mask[None, :], other=0.0)
b = tl.load(b_ptrs, mask=k_mask[:, None] & n_mask, other=0.0)
acc = tl.dot(a, b, acc)
d = acc.to(d_ptr.dtype.element_ty)
# Write back
d_ptrs = d_ptr + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, d, mask=n_mask)
def a_fused_m_grouped_bf16_gemm_nt_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
mappings: Tuple[torch.Tensor, torch.Tensor]):
m_indices, m_row_indices = mappings
r0, r1, r2 = b.shape
assert a.is_contiguous() and (b.is_contiguous() or b.mT.is_contiguous()) and d.is_contiguous()
assert m_indices.is_contiguous() and m_row_indices.is_contiguous()
assert a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16 and d.dtype == torch.bfloat16
assert m_indices.dtype == torch.int32 and m_row_indices.dtype == torch.int32
assert a.dim() == 2 and b.dim() == 3 and d.dim() == 2
assert a.size(1) == r2 and d.size(0) == m_indices.numel() and d.size(1) == r1
assert m_indices.numel() == m_row_indices.numel()
assert m_indices.numel() % get_mk_alignment_for_contiguous_layout() == 0
if d.size(0) == 0:
return d
M_, K = a.shape
B, K, N = r0, r2, r1
M = m_indices.numel()
grid = lambda meta: (triton.cdiv(M, meta['BLOCK_SIZE_M']) * triton.cdiv(N, meta['BLOCK_SIZE_N']), )
a_fused_m_grouped_bf16_gemm_contiguous_tl_impl[grid](a, b, d, m_indices, m_row_indices,
M, N, K, IS_B_K_MAJOR=b.is_contiguous())
def a_fused_m_grouped_bf16_gemm_nn_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
mappings: Tuple[torch.Tensor, torch.Tensor]):
a_fused_m_grouped_bf16_gemm_nt_contiguous_tl(a, b.mT, d, mappings)
@@ -0,0 +1,86 @@
import torch
import triton
import triton.language as tl
from typing import Tuple
from .tune_options import *
from .._C import get_mk_alignment_for_contiguous_layout
@triton.autotune(configs=get_k_grouped_gemm_configs(), key=[], restore_value=['d_ptr'])
@triton.jit
def b_fused_k_grouped_bf16_gemm_contiguous_tl_impl(a_ptr, b_ptr, d_ptr,
k_indices_ptr, k_start_ptr, k_end_ptr,
M: tl.constexpr,
N: tl.constexpr,
K,
ACC: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr):
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
pid_b = (pid // (num_pid_m * num_pid_n)).to(tl.int64)
pid = pid % (num_pid_m * num_pid_n)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + (pid % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
k_start = tl.load(k_start_ptr + pid_b)
k_end = tl.load(k_end_ptr + pid_b)
m_range = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
n_range = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
m_range = tl.max_contiguous(tl.multiple_of(m_range, BLOCK_SIZE_M), BLOCK_SIZE_M)
n_range = tl.max_contiguous(tl.multiple_of(n_range, BLOCK_SIZE_N), BLOCK_SIZE_N)
m_mask = (m_range < M)[:, None]
n_mask = (n_range < N)[None, :]
if k_start >= k_end:
if not ACC:
d_ptrs = d_ptr + pid_b * M * N + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=d_ptr.dtype.element_ty), mask=m_mask & n_mask)
return
# Compute
acc = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for k in range(k_start, k_end, BLOCK_SIZE_K):
k_range = (k + tl.arange(0, BLOCK_SIZE_K)).to(tl.int64)
rows = tl.load(k_indices_ptr + k_range).to(tl.int64)
a_ptrs = a_ptr + m_range[:, None] + k_range[None, :] * M
b_ptrs = b_ptr + rows[:, None] * N + n_range[None, :]
a = tl.load(a_ptrs, mask=m_mask, other=0.0)
b = tl.load(b_ptrs, mask=(rows >= 0)[:, None] & n_mask, other=0.0)
acc = tl.dot(a, b, acc)
d_ptrs = d_ptr + pid_b * M * N + m_range[:, None].to(tl.int64) * N + n_range[None, :]
if ACC:
acc += tl.load(d_ptrs, mask=m_mask & n_mask)
acc = acc.to(d_ptr.dtype.element_ty)
tl.store(d_ptrs, acc, mask=m_mask & n_mask)
def b_fused_k_grouped_bf16_gemm_tn_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
handle: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], acc: bool):
k_indices, k_start, k_end = handle
assert a.is_contiguous() and b.is_contiguous() and d.is_contiguous()
assert k_indices.is_contiguous() and k_start.is_contiguous() and k_end.is_contiguous()
assert a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
assert k_indices.dtype == torch.int32 and k_start.dtype == torch.int32 and k_end.dtype == torch.int32
assert a.dim() == 2 and b.dim() == 2 and d.dim() == 3
assert k_start.numel() == k_end.numel() and k_indices.size(0) == a.size(0)
assert d.size(0) == k_start.numel() and d.size(1) == a.size(1) and d.size(2) == b.size(1)
assert a.size(0) % get_mk_alignment_for_contiguous_layout() == 0
K, M = a.shape
K_, N = b.shape
B = k_start.numel()
grid = lambda META: (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N']) * B,)
b_fused_k_grouped_bf16_gemm_contiguous_tl_impl[grid](a, b, d, k_indices, k_start, k_end, M, N, K, ACC=acc)
+84
View File
@@ -0,0 +1,84 @@
import torch
import triton
import triton.language as tl
from typing import Tuple
from .tune_options import *
from .._C import get_mk_alignment_for_contiguous_layout
@triton.autotune(configs=get_m_grouped_gemm_configs(), key=[])
@triton.jit
def m_grouped_bf16_gemm_contiguous_tl_impl(a_ptr, b_ptr, d_ptr,
m_indices_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
BLOCK_SIZE_M: tl.constexpr,
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr,
IS_B_K_MAJOR: tl.constexpr):
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + (pid % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
m_range = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
n_range = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
n_mask = (n_range < N)[None, :]
# Empty tokens
batch_id = tl.load(m_indices_ptr + pid_m * BLOCK_SIZE_M).to(tl.int64)
if batch_id < 0:
d_ptrs = d_ptr + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=d_ptr.dtype.element_ty), mask=n_mask)
return
# Compute
a_ptrs = a_ptr + m_range[:, None].to(tl.int64) * K + tl.arange(0, BLOCK_SIZE_K)[None, :]
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
b_ptrs = b_ptr + batch_id * K * N + \
tl.arange(0, BLOCK_SIZE_K)[:, None].to(tl.int64) * (1 if IS_B_K_MAJOR else N) + \
n_range[None, :].to(tl.int64) * (K if IS_B_K_MAJOR else 1)
for k in range(0, K, BLOCK_SIZE_K):
k_mask = (k + tl.arange(0, BLOCK_SIZE_K)) < K
a = tl.load(a_ptrs, mask=k_mask[None, :], other=0.0)
b = tl.load(b_ptrs, mask=k_mask[:, None] & n_mask, other=0.0)
accumulator = tl.dot(a, b, accumulator)
a_ptrs += BLOCK_SIZE_K
b_ptrs += BLOCK_SIZE_K * (1 if IS_B_K_MAJOR else N)
# Write back
d_ptrs = d_ptr + m_range[:, None].to(tl.int64) * N + n_range[None, :]
tl.store(d_ptrs, accumulator.to(d_ptr.dtype.element_ty), mask=n_mask)
def m_grouped_bf16_gemm_nt_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
m_indices: torch.Tensor):
r0, r1, r2 = b.shape
assert a.is_contiguous() and (b.is_contiguous or b.mT.is_contiguous())
assert m_indices.is_contiguous() and d.is_contiguous()
assert a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
assert m_indices.dtype == torch.int32 and d.dtype == torch.bfloat16
assert a.dim() == 2 and b.dim() == 3 and d.dim() == 2
assert a.size(1) == r2 and a.size(0) == d.size(0) and r1 == d.size(1)
assert m_indices.numel() == a.size(0)
assert a.size(0) % get_mk_alignment_for_contiguous_layout() == 0
M, K = a.shape
B, N, K_ = r0, r1, r2
# For Triton 2.0, persistent kernel will lead to errors
grid = lambda META: (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N']), )
m_grouped_bf16_gemm_contiguous_tl_impl[grid](
a, b, d, m_indices, M, N, K, IS_B_K_MAJOR=b.is_contiguous())
def m_grouped_bf16_gemm_nn_contiguous_tl(a: torch.Tensor, b: torch.Tensor, d: torch.Tensor,
m_indices: torch.Tensor):
m_grouped_bf16_gemm_nt_contiguous_tl(a, b.mT, d, m_indices)
+28
View File
@@ -0,0 +1,28 @@
from triton import Config
from .._C import get_mk_alignment_for_contiguous_layout
def get_config_smem_size(config: Config, elem_bytes: int = 2):
# NOTES: FP8 kernels will not use Triton, so by default we assume BF16 kernels
return (config.kwargs['BLOCK_SIZE_M'] + config.kwargs['BLOCK_SIZE_N']) * config.kwargs['BLOCK_SIZE_K'] * elem_bytes * config.num_stages
_gemm_configs = [
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8),
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=8),
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8),
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=8),
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=8),
Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=4, num_warps=8),
Config({'BLOCK_SIZE_M': 64, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=2, num_warps=4),
Config({'BLOCK_SIZE_M': 32, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=4),
Config({'BLOCK_SIZE_M': 32, 'BLOCK_SIZE_N': 64, 'BLOCK_SIZE_K': 128, 'GROUP_SIZE_M': 8}, num_stages=3, num_warps=4),
]
# NOTES: we only consider A100 shared memory sizes here, as legacy kernels are only used for Ampere
_gemm_configs = list(filter(lambda x: get_config_smem_size(x) <= 166912, _gemm_configs))
_gemm_configs = list(filter(lambda x: x.kwargs['BLOCK_SIZE_M'] <= get_mk_alignment_for_contiguous_layout(), _gemm_configs))
_gemm_configs = list(filter(lambda x: x.kwargs['BLOCK_SIZE_K'] <= get_mk_alignment_for_contiguous_layout(), _gemm_configs))
get_m_grouped_gemm_configs = lambda: list(filter(lambda x: x.kwargs['BLOCK_SIZE_M'] <= get_mk_alignment_for_contiguous_layout(), _gemm_configs))
get_k_grouped_gemm_configs = lambda: list(filter(lambda x: x.kwargs['BLOCK_SIZE_K'] <= get_mk_alignment_for_contiguous_layout(), _gemm_configs))