chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add FP8 Blockwise GEMM Backend Flag --fp8-gemm-backend (#14379)

Browse Source
This commit is contained in:
b8zhong
2025-12-09 12:05:56 -08:00
committed by GitHub
parent 73df7a4e8d
commit 55504df2f7
10 changed files with 306 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/bench_one_batch.py
View File

@@ -67,6 +67,7 @@ from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.distributed.parallel_state import destroy_distributed_environment
from sglang.srt.entrypoints.engine import _set_envs_and_config
from sglang.srt.layers.moe import initialize_moe_config
from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.managers.scheduler_dp_attn_mixin import prepare_mlp_sync_batch_raw
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -631,6 +632,7 @@ def latency_test(
tp_rank,
):
initialize_moe_config(server_args)
initialize_fp8_gemm_config(server_args)
# Set CPU affinity
if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):