From 87699d48eb6935d37415c2e19d107b9880e415df Mon Sep 17 00:00:00 2001 From: Douglas Yang Date: Sun, 4 Jan 2026 12:17:08 -0800 Subject: [PATCH] fix: only publish trace from tp 0 (#16411) --- python/sglang/test/nightly_bench_utils.py | 17 +++++++++++++++-- scripts/ci/publish_traces.py | 10 +++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/sglang/test/nightly_bench_utils.py b/python/sglang/test/nightly_bench_utils.py index 38eb61c2b..0e9074cc1 100644 --- a/python/sglang/test/nightly_bench_utils.py +++ b/python/sglang/test/nightly_bench_utils.py @@ -132,12 +132,25 @@ def save_results_as_pydantic_models( profile_link_decode = None if res.profile_link: + # Collect all trace files, preferring TP-0 to match upload behavior + # (only TP-0 traces are published to avoid duplicates) + extend_files = [] + decode_files = [] for file in os.listdir(res.profile_link): if file.endswith(".trace.json.gz") or file.endswith(".trace.json"): if "extend" in file.lower() or "prefill" in file.lower(): - profile_link_extend = os.path.join(res.profile_link, file) + extend_files.append(file) elif "decode" in file.lower(): - profile_link_decode = os.path.join(res.profile_link, file) + decode_files.append(file) + + # Sort to prefer TP-0 files (TP-0 < TP-1 < TP-2... alphabetically) + extend_files.sort() + decode_files.sort() + + if extend_files: + profile_link_extend = os.path.join(res.profile_link, extend_files[0]) + if decode_files: + profile_link_decode = os.path.join(res.profile_link, decode_files[0]) benchmark_result = BenchmarkResult( model_path=model_path, diff --git a/scripts/ci/publish_traces.py b/scripts/ci/publish_traces.py index 9805f55cb..6a630146f 100644 --- a/scripts/ci/publish_traces.py +++ b/scripts/ci/publish_traces.py @@ -266,7 +266,10 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retr def copy_trace_files(source_dir, target_base_path): - """Copy trace files and return list of files to upload""" + """Copy trace files and return list of files to upload. + + Only uploads traces from TP rank 0 to avoid duplicated data across tensor parallel ranks. + """ files_to_upload = [] if not os.path.exists(source_dir): @@ -277,6 +280,11 @@ def copy_trace_files(source_dir, target_base_path): for root, dirs, files in os.walk(source_dir): for file in files: if file.endswith(".json.gz"): + + # Only upload TP rank 0 traces to avoid duplicates across tensor parallel ranks + if "TP-" in file and "TP-0" not in file: + continue + source_file = os.path.join(root, file) # Calculate relative path from source_dir rel_path = os.path.relpath(source_file, source_dir)