chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[AMD] support two batch overlapping for mori ep (#17953)

Browse Source 
Co-authored-by: kkHuang-amd <wunhuang@amd.com>
Co-authored-by: Feiyue Zhai <feiyue.zhai@amd.com>
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
billishyahao
2026-02-21 00:45:55 +08:00
committed by GitHub
parent 38ee749dd9
commit fbb6098487
11 changed files with 725 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
python/sglang/test/bench_one_batch_server_internal.py
View File

@@ -749,22 +749,34 @@ def run_benchmark_internal(
else:
tokenizer = get_tokenizer(tokenizer_path)
# Get token capacity
internal_state = server_info.get("internal_states", [{}])
skip_token_capacity_threshold = (
internal_state[0].get("memory_usage", {}).get("token_capacity", 1000000000)
)
dp_size = internal_state[0].get("dp_size", None) or 1
# Get effective max running requests
max_running_requests_per_dp = internal_state[0].get(
"effective_max_running_requests_per_dp", -1
)
dp_size = server_info.get("dp_size", None) or 1
# Get token capacity
skip_token_capacity_threshold = 0
for i in range(dp_size):
skip_token_capacity_threshold += (
internal_state[i]
.get("memory_usage", {})
.get("token_capacity", 1000000000)
)
assert (
max_running_requests_per_dp > 0
), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}"
skip_max_running_requests_threshold = max_running_requests_per_dp * dp_size
print(f"{max_running_requests_per_dp=}")
print(f"{dp_size=}")
print(f"{skip_max_running_requests_threshold=}")
print(f"{skip_token_capacity_threshold=}")
# Warmup
if not bench_args.skip_warmup:
print("=" * 8 + " Warmup Begin " + "=" * 8)