[diffusion] amd: fix SGLANG_DIFFUSION_ATTENTION_BACKEND env var for diffusion attention backend selection (#16325)
Co-authored-by: root <root@mi300x8-008.atl1.do.cpe.ice.amd.com>
This commit is contained in:
@@ -11,7 +11,6 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import sglang.multimodal_gen.envs as envs
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import (
|
||||
AttentionBackendEnum,
|
||||
DeviceCapability,
|
||||
@@ -93,11 +92,6 @@ class RocmPlatform(Platform):
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> str:
|
||||
logger.info(
|
||||
"Trying SGLANG_DIFFUSION_ATTENTION_BACKEND=%s",
|
||||
envs.SGLANG_DIFFUSION_ATTENTION_BACKEND,
|
||||
)
|
||||
|
||||
if selected_backend == AttentionBackendEnum.TORCH_SDPA:
|
||||
logger.info("Using Torch SDPA backend.")
|
||||
return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend"
|
||||
@@ -108,13 +102,10 @@ class RocmPlatform(Platform):
|
||||
elif selected_backend == AttentionBackendEnum.AITER:
|
||||
if dtype not in (torch.float16, torch.bfloat16):
|
||||
logger.warning(
|
||||
"AITer backend only supports fp16/bf16 inputs but got dtype=%s. "
|
||||
"Falling back to Torch SDPA backend.",
|
||||
"AITer backend works best with fp16/bf16 inputs but got dtype=%s. "
|
||||
"Proceeding with AITer anyway.",
|
||||
dtype,
|
||||
)
|
||||
# TODO: need to compare triton with sdpa as an alternative backend
|
||||
return "sglang.multimodal_gen.runtime.layers.attention.backends.sdpa.SDPABackend"
|
||||
|
||||
logger.info("Using AITer backend on ROCm.")
|
||||
return "sglang.multimodal_gen.runtime.layers.attention.backends.aiter.AITerBackend"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user