fix: only publish trace from tp 0 (#16411)
This commit is contained in:
@@ -266,7 +266,10 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retr
|
||||
|
||||
|
||||
def copy_trace_files(source_dir, target_base_path):
|
||||
"""Copy trace files and return list of files to upload"""
|
||||
"""Copy trace files and return list of files to upload.
|
||||
|
||||
Only uploads traces from TP rank 0 to avoid duplicated data across tensor parallel ranks.
|
||||
"""
|
||||
files_to_upload = []
|
||||
|
||||
if not os.path.exists(source_dir):
|
||||
@@ -277,6 +280,11 @@ def copy_trace_files(source_dir, target_base_path):
|
||||
for root, dirs, files in os.walk(source_dir):
|
||||
for file in files:
|
||||
if file.endswith(".json.gz"):
|
||||
|
||||
# Only upload TP rank 0 traces to avoid duplicates across tensor parallel ranks
|
||||
if "TP-" in file and "TP-0" not in file:
|
||||
continue
|
||||
|
||||
source_file = os.path.join(root, file)
|
||||
# Calculate relative path from source_dir
|
||||
rel_path = os.path.relpath(source_file, source_dir)
|
||||
|
||||
Reference in New Issue
Block a user