[diffusion] improve: improve qwen-image-edit performance to align with LightX2V (#15812)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -129,6 +129,94 @@ def fuse_scale_shift_kernel_blc_opt(
|
||||
tl.store(y_ptr + x_off, y, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fuse_scale_shift_gate_select01_kernel_blc_opt(
|
||||
x_ptr,
|
||||
shift0_ptr,
|
||||
scale0_ptr,
|
||||
gate0_ptr,
|
||||
shift1_ptr,
|
||||
scale1_ptr,
|
||||
gate1_ptr,
|
||||
index_ptr,
|
||||
y_ptr,
|
||||
gate_out_ptr,
|
||||
B,
|
||||
L,
|
||||
C,
|
||||
stride_x_b,
|
||||
stride_x_l,
|
||||
stride_x_c,
|
||||
stride_s0_b,
|
||||
stride_s0_c,
|
||||
stride_sc0_b,
|
||||
stride_sc0_c,
|
||||
stride_g0_b,
|
||||
stride_g0_c,
|
||||
stride_s1_b,
|
||||
stride_s1_c,
|
||||
stride_sc1_b,
|
||||
stride_sc1_c,
|
||||
stride_g1_b,
|
||||
stride_g1_c,
|
||||
stride_i_b,
|
||||
stride_i_l,
|
||||
stride_go_b,
|
||||
stride_go_l,
|
||||
stride_go_c,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_C: tl.constexpr,
|
||||
):
|
||||
pid_l = tl.program_id(0)
|
||||
pid_c = tl.program_id(1)
|
||||
pid_b = tl.program_id(2)
|
||||
|
||||
l_offsets = pid_l * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
c_offsets = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
|
||||
|
||||
mask_l = l_offsets < L
|
||||
mask_c = c_offsets < C
|
||||
mask = mask_l[:, None] & mask_c[None, :]
|
||||
|
||||
x_off = (
|
||||
pid_b * stride_x_b
|
||||
+ l_offsets[:, None] * stride_x_l
|
||||
+ c_offsets[None, :] * stride_x_c
|
||||
)
|
||||
x = tl.load(x_ptr + x_off, mask=mask, other=0)
|
||||
|
||||
idx_off = pid_b * stride_i_b + l_offsets * stride_i_l
|
||||
idx = tl.load(index_ptr + idx_off, mask=mask_l, other=0).to(tl.int1)[:, None]
|
||||
|
||||
s0_off = pid_b * stride_s0_b + c_offsets[None, :] * stride_s0_c
|
||||
sc0_off = pid_b * stride_sc0_b + c_offsets[None, :] * stride_sc0_c
|
||||
g0_off = pid_b * stride_g0_b + c_offsets[None, :] * stride_g0_c
|
||||
s1_off = pid_b * stride_s1_b + c_offsets[None, :] * stride_s1_c
|
||||
sc1_off = pid_b * stride_sc1_b + c_offsets[None, :] * stride_sc1_c
|
||||
g1_off = pid_b * stride_g1_b + c_offsets[None, :] * stride_g1_c
|
||||
|
||||
shift0 = tl.load(shift0_ptr + s0_off, mask=mask_c[None, :], other=0)
|
||||
scale0 = tl.load(scale0_ptr + sc0_off, mask=mask_c[None, :], other=0)
|
||||
gate0 = tl.load(gate0_ptr + g0_off, mask=mask_c[None, :], other=0)
|
||||
shift1 = tl.load(shift1_ptr + s1_off, mask=mask_c[None, :], other=0)
|
||||
scale1 = tl.load(scale1_ptr + sc1_off, mask=mask_c[None, :], other=0)
|
||||
gate1 = tl.load(gate1_ptr + g1_off, mask=mask_c[None, :], other=0)
|
||||
|
||||
shift = tl.where(idx, shift1, shift0)
|
||||
scale = tl.where(idx, scale1, scale0)
|
||||
gate = tl.where(idx, gate1, gate0)
|
||||
|
||||
y = x * (1 + scale) + shift
|
||||
tl.store(y_ptr + x_off, y, mask=mask)
|
||||
|
||||
go_off = (
|
||||
pid_b * stride_go_b
|
||||
+ l_offsets[:, None] * stride_go_l
|
||||
+ c_offsets[None, :] * stride_go_c
|
||||
)
|
||||
tl.store(gate_out_ptr + go_off, gate, mask=mask)
|
||||
|
||||
|
||||
def fuse_scale_shift_kernel(
|
||||
x: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
@@ -241,6 +329,78 @@ def fuse_scale_shift_kernel(
|
||||
return output
|
||||
|
||||
|
||||
def fuse_scale_shift_gate_select01_kernel(
|
||||
x: torch.Tensor,
|
||||
scale0: torch.Tensor,
|
||||
shift0: torch.Tensor,
|
||||
gate0: torch.Tensor,
|
||||
scale1: torch.Tensor,
|
||||
shift1: torch.Tensor,
|
||||
gate1: torch.Tensor,
|
||||
index: torch.Tensor,
|
||||
block_l: int = 128,
|
||||
block_c: int = 128,
|
||||
):
|
||||
assert x.is_contiguous()
|
||||
B, L, C = x.shape
|
||||
output = torch.empty_like(x)
|
||||
gate_out = torch.empty_like(x)
|
||||
|
||||
if (
|
||||
scale0.dim() != 2
|
||||
or shift0.dim() != 2
|
||||
or gate0.dim() != 2
|
||||
or scale1.dim() != 2
|
||||
or shift1.dim() != 2
|
||||
or gate1.dim() != 2
|
||||
):
|
||||
raise ValueError("scale0/shift0/gate0/scale1/shift1/gate1 must be 2D [B, C]")
|
||||
if index.dim() != 2:
|
||||
raise ValueError("index must be 2D [B, L]")
|
||||
|
||||
grid = (triton.cdiv(L, block_l), triton.cdiv(C, block_c), B)
|
||||
fuse_scale_shift_gate_select01_kernel_blc_opt[grid](
|
||||
x,
|
||||
shift0,
|
||||
scale0,
|
||||
gate0,
|
||||
shift1,
|
||||
scale1,
|
||||
gate1,
|
||||
index,
|
||||
output,
|
||||
gate_out,
|
||||
B,
|
||||
L,
|
||||
C,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
shift0.stride(0),
|
||||
shift0.stride(1),
|
||||
scale0.stride(0),
|
||||
scale0.stride(1),
|
||||
gate0.stride(0),
|
||||
gate0.stride(1),
|
||||
shift1.stride(0),
|
||||
shift1.stride(1),
|
||||
scale1.stride(0),
|
||||
scale1.stride(1),
|
||||
gate1.stride(0),
|
||||
gate1.stride(1),
|
||||
index.stride(0),
|
||||
index.stride(1),
|
||||
gate_out.stride(0),
|
||||
gate_out.stride(1),
|
||||
gate_out.stride(2),
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_C=block_c,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return output, gate_out
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK_HS_HALF": 32}, num_warps=2),
|
||||
|
||||
Reference in New Issue
Block a user