chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Auto Sync] Update grpc_request_manager.py, tokenizer_manag... (20260214) (#18838)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Lianmin Zheng
2026-02-14 18:12:32 -08:00
committed by GitHub
parent 190fa8246f
commit b33769786f
7 changed files with 544 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

335
scripts/code_sync/check_commits.py
View File

@@ -2,13 +2,15 @@
List commits in the private repo that need to be synced to the OSS repo.
NOTE:
1. You need to execute this script in the git root folder.
1. This script resolves the git root automatically and can be run anywhere
inside the repo.
This script will:
1. Find the most recent sync commit (message starts with
"[Automated PR] Copy OSS code from commit").
2. Scan commits after that point and keep those that touch the configured paths.
3. Print a markdown summary with commit links and write it to GitHub Step Summary.
3. Compare added diff lines in relevant files against OSS main.
4. Print a markdown summary with commit links and write it to GitHub Step Summary.
Usage:
python3 scripts/code_sync/check_commits.py
@@ -18,41 +20,35 @@ import argparse
import os
import shutil
import subprocess
from typing import List, Optional, Tuple
import sys
from dataclasses import dataclass
from typing import Dict, List, Optional, Set, Tuple
# Allow sibling imports regardless of the working directory.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from utils import ( # noqa: E402
FOLDER_NAMES,
get_last_sync_commit,
write_github_step_summary,
)
# --- Configuration Begin ---
# List of folders and files to copy to the OSS repo.
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
private_repo = "your-org/sglang-private-repo"
sync_commit_prefix = r"\[Automated PR\] Copy OSS code from commit"
oss_repo_url = "https://github.com/sgl-project/sglang.git"
oss_repo_branch = "main"
default_oss_repo_dir = ".oss_repo"
# --- Configuration End ---
def write_github_step_summary(content: str) -> None:
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
@dataclass
class CommitInfo:
commit_hash: str
subject: str
commit_date: str
relevant_files: List[str]
synced_lines: int
total_added_lines: int
def check_dependencies() -> None:
@@ -61,7 +57,23 @@ def check_dependencies() -> None:
raise EnvironmentError("git is not installed or not in PATH.")
def get_repo_from_origin() -> str:
def get_repo_root() -> str:
try:
output = subprocess.run(
["git", "rev-parse", "--show-toplevel"],
capture_output=True,
text=True,
check=True,
).stdout.strip()
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Unable to determine git repo root: {e.stderr or e}") from e
if not output:
raise RuntimeError("Unable to determine git repo root.")
return os.path.abspath(output)
def get_repo_from_origin(repo_root: str) -> str:
"""Try to infer the repo slug (owner/name) from git remote.origin.url."""
try:
url = subprocess.run(
@@ -69,6 +81,7 @@ def get_repo_from_origin() -> str:
capture_output=True,
text=True,
check=True,
cwd=repo_root,
).stdout.strip()
except subprocess.CalledProcessError:
return private_repo
@@ -85,29 +98,48 @@ def get_repo_from_origin() -> str:
return repo or private_repo
def get_last_sync_commit() -> Optional[str]:
"""Find the most recent sync commit that copied from OSS."""
try:
result = subprocess.run(
[
"git",
"log",
"-1",
"--grep",
sync_commit_prefix,
"--format=%H",
],
capture_output=True,
text=True,
def get_default_oss_repo_path(repo_root: str) -> str:
env_path = os.environ.get("OSS_REPO_PATH")
if env_path:
return os.path.abspath(env_path)
return os.path.abspath(os.path.join(repo_root, default_oss_repo_dir))
def ensure_oss_repo(oss_repo_path: str, repo_url: str, branch: str) -> str:
oss_repo_path = os.path.abspath(oss_repo_path)
if os.path.exists(oss_repo_path) and not os.path.isdir(oss_repo_path):
raise RuntimeError(f"OSS repo path is not a directory: {oss_repo_path}")
if os.path.isdir(os.path.join(oss_repo_path, ".git")):
try:
subprocess.run(
["git", "-C", oss_repo_path, "rev-parse", "--is-inside-work-tree"],
capture_output=True,
text=True,
check=True,
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"OSS repo path exists but is not a git repo: {oss_repo_path}"
) from e
subprocess.run(
["git", "-C", oss_repo_path, "fetch", "origin", branch, "--depth", "1"],
check=True,
).stdout.strip()
return result or None
except subprocess.CalledProcessError as e:
print(f"Error finding last sync commit: {e.stderr}")
return None
)
return oss_repo_path
parent_dir = os.path.dirname(oss_repo_path)
if parent_dir and not os.path.isdir(parent_dir):
os.makedirs(parent_dir, exist_ok=True)
subprocess.run(
["git", "clone", "--depth", "1", "--branch", branch, repo_url, oss_repo_path],
check=True,
)
return oss_repo_path
def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
def get_commits_since(repo_root: str, last_sync_hash: Optional[str]) -> List[str]:
"""Get commit hashes from last sync commit (exclusive) to HEAD."""
try:
if last_sync_hash:
@@ -115,7 +147,7 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
else:
command = ["git", "rev-list", "HEAD"]
result = subprocess.run(
command, capture_output=True, text=True, check=True
command, capture_output=True, text=True, check=True, cwd=repo_root
).stdout.strip()
return [line for line in result.split("\n") if line]
except subprocess.CalledProcessError as e:
@@ -123,13 +155,14 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]:
return []
def get_changed_files(commit_hash: str) -> List[str]:
def get_changed_files(repo_root: str, commit_hash: str) -> List[str]:
try:
output = subprocess.run(
["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
capture_output=True,
text=True,
check=True,
cwd=repo_root,
).stdout.strip()
return [line for line in output.split("\n") if line]
except subprocess.CalledProcessError as e:
@@ -147,11 +180,115 @@ def get_relevant_files(changed_files: List[str]) -> List[str]:
return [
changed_file
for changed_file in changed_files
if any(is_relevant_path(changed_file, path) for path in folder_names)
if any(is_relevant_path(changed_file, path) for path in FOLDER_NAMES)
]
def get_commit_summary(commit_hash: str) -> Tuple[str, str]:
def get_added_lines_by_file(
repo_root: str, commit_hash: str, relevant_files: List[str]
) -> Dict[str, List[str]]:
if not relevant_files:
return {}
command = [
"git",
"show",
"--no-color",
"--unified=0",
"--format=",
commit_hash,
"--",
] + relevant_files
try:
output = subprocess.run(
command, capture_output=True, text=True, check=True, cwd=repo_root
).stdout
except subprocess.CalledProcessError as e:
print(f"Error getting diff for {commit_hash}: {e.stderr}")
return {}
added_lines: Dict[str, List[str]] = {path: [] for path in relevant_files}
relevant_set = set(relevant_files)
current_file: Optional[str] = None
for line in output.splitlines():
if line.startswith("diff --git "):
current_file = None
continue
if line.startswith("+++ "):
file_path = None
if line.startswith("+++ b/"):
file_path = line[6:]
else:
candidate = line[4:]
if candidate == "/dev/null":
file_path = None
elif candidate.startswith("b/") or candidate.startswith("a/"):
file_path = candidate[2:]
else:
file_path = candidate
if file_path in relevant_set:
current_file = file_path
else:
current_file = None
continue
if current_file and line.startswith("+") and not line.startswith("+++ "):
added_lines[current_file].append(line[1:])
return added_lines
def get_oss_file_lines(
oss_repo_path: str,
oss_ref: str,
file_path: str,
cache: Dict[str, Optional[Set[str]]],
) -> Optional[Set[str]]:
if file_path in cache:
return cache[file_path]
try:
output = subprocess.run(
["git", "-C", oss_repo_path, "show", f"{oss_ref}:{file_path}"],
capture_output=True,
text=True,
errors="replace",
check=True,
).stdout
except subprocess.CalledProcessError:
cache[file_path] = None
return None
lines = output.splitlines()
line_set = set(lines)
cache[file_path] = line_set
return line_set
def count_synced_lines(
added_lines_by_file: Dict[str, List[str]],
oss_repo_path: str,
oss_ref: str,
oss_file_cache: Dict[str, Optional[Set[str]]],
) -> Tuple[int, int]:
total_added_lines = 0
synced_lines = 0
for file_path, lines in added_lines_by_file.items():
total_added_lines += len(lines)
if not lines:
continue
oss_lines = get_oss_file_lines(
oss_repo_path, oss_ref, file_path, oss_file_cache
)
if not oss_lines:
continue
for line in lines:
if line in oss_lines:
synced_lines += 1
return synced_lines, total_added_lines
def get_commit_summary(repo_root: str, commit_hash: str) -> Tuple[str, str]:
"""Return (subject, date) for a commit."""
try:
output = subprocess.run(
@@ -159,6 +296,7 @@ def get_commit_summary(commit_hash: str) -> Tuple[str, str]:
capture_output=True,
text=True,
check=True,
cwd=repo_root,
).stdout.strip()
subject, commit_date = output.split("\x00", 1)
except subprocess.CalledProcessError as e:
@@ -195,13 +333,20 @@ def format_commit_block(
commit_hash: str,
commit_date: str,
relevant_files: List[str],
synced_lines: int,
total_added_lines: int,
) -> str:
short_hash = commit_hash[:9]
commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
files_str = format_files_list(relevant_files) if relevant_files else "- None"
status_icon = "✅" if synced_lines == total_added_lines else "❌"
status_line = (
f"status: {status_icon} {synced_lines}/{total_added_lines} lines synced"
)
return "\n".join(
[
f"#### {subject}",
status_line,
f"date: {commit_date}",
"files to sync:",
files_str,
@@ -215,7 +360,7 @@ def format_commit_block(
def format_output(
repo: str,
last_sync: Optional[Tuple[str, str, str]],
commits: List[Tuple[str, str, str, List[str]]],
commits: List[CommitInfo],
) -> str:
lines: List[str] = []
if last_sync:
@@ -229,9 +374,17 @@ def format_output(
lines.append("No commits need to be synced.")
return "\n".join(lines) + "\n"
for commit_hash, subject, commit_date, relevant_files in commits:
for commit in commits:
lines.append(
format_commit_block(repo, subject, commit_hash, commit_date, relevant_files)
format_commit_block(
repo,
commit.subject,
commit.commit_hash,
commit.commit_date,
commit.relevant_files,
commit.synced_lines,
commit.total_added_lines,
)
)
return "\n".join(lines)
@@ -247,30 +400,78 @@ def main() -> None:
default=0,
help="Limit number of commits printed (0 means no limit).",
)
parser.add_argument(
"--oss-repo-path",
default=None,
help="Path to OSS repo clone (default: $OSS_REPO_PATH or .oss_repo).",
)
parser.add_argument(
"--oss-repo-url",
default=oss_repo_url,
help="OSS repo URL (default: https://github.com/sgl-project/sglang.git).",
)
parser.add_argument(
"--oss-branch",
default=oss_repo_branch,
help="OSS repo branch to check (default: main).",
)
args = parser.parse_args()
check_dependencies()
repo_root = get_repo_root()
oss_repo_path = (
os.path.abspath(args.oss_repo_path)
if args.oss_repo_path
else get_default_oss_repo_path(repo_root)
)
repo = get_repo_from_origin()
last_sync_hash = get_last_sync_commit()
repo = get_repo_from_origin(repo_root)
last_sync_hash = get_last_sync_commit(repo_root)
last_sync_block = None
if last_sync_hash:
last_sync_subject, last_sync_date = get_commit_summary(last_sync_hash)
last_sync_subject, last_sync_date = get_commit_summary(
repo_root, last_sync_hash
)
last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date)
commits = get_commits_since(last_sync_hash)
commits = get_commits_since(repo_root, last_sync_hash)
if args.limit > 0:
commits = commits[: args.limit]
relevant_commits: List[Tuple[str, str, str, List[str]]] = []
relevant_commit_inputs: List[Tuple[str, List[str]]] = []
for commit_hash in commits:
changed_files = get_changed_files(commit_hash)
changed_files = get_changed_files(repo_root, commit_hash)
if not changed_files:
continue
relevant_files = get_relevant_files(changed_files)
if relevant_files:
subject, commit_date = get_commit_summary(commit_hash)
relevant_commits.append((commit_hash, subject, commit_date, relevant_files))
relevant_commit_inputs.append((commit_hash, relevant_files))
relevant_commits: List[CommitInfo] = []
if relevant_commit_inputs:
oss_repo_path = ensure_oss_repo(
oss_repo_path, args.oss_repo_url, args.oss_branch
)
oss_ref = f"origin/{args.oss_branch}"
oss_file_cache: Dict[str, Optional[Set[str]]] = {}
for commit_hash, relevant_files in relevant_commit_inputs:
subject, commit_date = get_commit_summary(repo_root, commit_hash)
added_lines_by_file = get_added_lines_by_file(
repo_root, commit_hash, relevant_files
)
synced_lines, total_added_lines = count_synced_lines(
added_lines_by_file, oss_repo_path, oss_ref, oss_file_cache
)
relevant_commits.append(
CommitInfo(
commit_hash=commit_hash,
subject=subject,
commit_date=commit_date,
relevant_files=relevant_files,
synced_lines=synced_lines,
total_added_lines=total_added_lines,
)
)
output = format_output(repo, last_sync_block, relevant_commits)
print(output)

44
scripts/code_sync/copy_from_oss.py
View File

@@ -31,45 +31,19 @@ import datetime
import os
import shutil
import subprocess
import sys
import tempfile
# --- Configuration Begin ---
# List of folders and files to copy from the OSS repo.
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
# Allow sibling imports regardless of the working directory.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from utils import FOLDER_NAMES, write_github_step_summary # noqa: E402
# --- Configuration Begin ---
private_repo = "your-org/sglang-private-repo"
# --- Configuration End ---
def write_github_step_summary(content):
if not os.environ.get("GITHUB_STEP_SUMMARY"):
return
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
def check_dependencies():
"""Check for required command-line tools."""
if not shutil.which("git"):
@@ -145,10 +119,10 @@ def get_source_folder(args):
return oss_root, temp_dir, commit_hash
def sync_directories(oss_root, folder_names, dry_run):
def sync_directories(oss_root, sync_paths, dry_run):
"""Sync specified directories from oss_root to current working directory."""
rsync_commands = []
for folder_name in folder_names:
for folder_name in sync_paths:
target_name = f"{oss_root}/{folder_name}"
src_name = "./" + "/".join(folder_name.split("/")[:-1])
cmd = f"rsync -r --delete {target_name} {src_name}"
@@ -259,7 +233,7 @@ def main():
try:
# Sync directories
sync_directories(oss_root, folder_names, args.dry_run)
sync_directories(oss_root, FOLDER_NAMES, args.dry_run)
# Check for changes and create PR if necessary
if not check_for_changes():

195
scripts/code_sync/copy_to_oss.py
View File

@@ -29,44 +29,20 @@ python3 scripts/copy_to_oss.py --commit <commit_hash> --dry-run
import argparse
import datetime
import os
import re
import shutil
import subprocess
import sys
import tempfile
# --- Configuration Begin ---
# List of folders and files to copy to the OSS repo.
# Changes outside these paths will be ignored.
folder_names = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
# Allow sibling imports regardless of the working directory.
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
# --- Configuration End ---
def write_github_step_summary(content):
if not os.environ.get("GITHUB_STEP_SUMMARY"):
return
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
f.write(content)
from utils import ( # noqa: E402
FOLDER_NAMES,
find_latest_oss_sync_commit,
write_github_step_summary,
)
def get_commit_info(commit_ref):
@@ -131,7 +107,7 @@ def create_filtered_patch(commit_hash, dry_run):
# Filter the list of files
relevant_files = [
f for f in changed_files if any(f.startswith(path) for path in folder_names)
f for f in changed_files if any(f.startswith(path) for path in FOLDER_NAMES)
]
if not relevant_files:
@@ -190,7 +166,7 @@ def get_oss_repo(dry_run):
print(f"\nCreated temporary directory for OSS repo: {temp_dir}")
repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git"
command = ["git", "clone", "--branch", "main", repo_url, oss_root]
command = ["git", "clone", repo_url, oss_root]
print(f"Run: {' '.join(command)}")
if not dry_run:
@@ -209,8 +185,9 @@ def _apply_patch(patch_file, dry_run):
"""
Try to apply a patch, falling back to --3way merge if a clean apply fails.
Returns True if the patch was applied successfully (clean or via --3way),
or raises an exception with full diagnostics if both methods fail.
Returns True if the patch was applied cleanly.
Returns False if conflicts were encountered (changes are still staged
with conflict markers so a PR can be created for manual resolution).
"""
# --- Attempt 1: clean git apply ---
apply_cmd = ["git", "apply", patch_file]
@@ -235,8 +212,8 @@ def _apply_patch(patch_file, dry_run):
print("✅ Patch applied via --3way merge (no conflicts).")
return True
# --- Both attempts failed — gather diagnostics ---
print(f"❌ --3way merge also failed:\n{result_3way.stderr.strip()}\n")
# --- --3way left conflict markers in the working tree ---
print(f"⚠️ --3way merge had conflicts:\n{result_3way.stderr.strip()}\n")
# Show which hunks conflict
check_cmd = ["git", "apply", "--check", "--verbose", patch_file]
@@ -272,7 +249,7 @@ def _apply_patch(patch_file, dry_run):
# Write a rich summary to the GitHub Actions step summary
summary_lines = [
"\n## ❌ Patch could not be applied automatically\n",
"\n## ⚠️ Patch had conflicts — PR created for manual resolution\n",
"### Conflict details\n",
f"```\n{conflict_details}\n```\n",
]
@@ -287,18 +264,22 @@ def _apply_patch(patch_file, dry_run):
)
write_github_step_summary("".join(summary_lines))
# Reset the working tree so the finally-block cleanup is safe
subprocess.run(["git", "checkout", "."], capture_output=True, text=True)
raise RuntimeError(
"Patch could not be applied. See the CI log and GitHub Actions "
"step summary for the full patch and conflict details."
)
return False
def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run):
def apply_patch_and_push(
oss_root, patch_file, branch_name, commit_message, base_oss_commit, dry_run
):
"""
In the OSS repo, create a branch, apply the patch, commit, and push.
In the OSS repo, create a branch from base_oss_commit, apply the patch,
commit, and push.
Args:
base_oss_commit: The OSS commit hash to branch from (the last sync
point). If None, the current HEAD (main) is used.
Returns True if the patch applied cleanly, False if there were conflicts
(the conflicted state is still committed and pushed so a PR can be opened).
"""
print("\nApplying patch and pushing to OSS repo...")
@@ -306,15 +287,19 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
if not dry_run:
os.chdir(oss_root)
applied_cleanly = True
try:
# Create a new branch
checkout_cmd = ["git", "checkout", "-b", branch_name]
# Check out a new branch from the base OSS commit
if base_oss_commit:
checkout_cmd = ["git", "checkout", "-b", branch_name, base_oss_commit]
else:
checkout_cmd = ["git", "checkout", "-b", branch_name]
print(f"Run: {' '.join(checkout_cmd)}")
if not dry_run:
subprocess.run(checkout_cmd, check=True, capture_output=True, text=True)
# Apply the patch (with --3way fallback and diagnostics)
_apply_patch(patch_file, dry_run)
applied_cleanly = _apply_patch(patch_file, dry_run)
# Configure git user and stage changes
post_apply_commands = [
@@ -359,7 +344,15 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_
if not dry_run:
os.chdir(original_cwd)
print("✅ Branch created, patch applied, and pushed successfully.")
if applied_cleanly:
print("✅ Branch created, patch applied cleanly, and pushed successfully.")
else:
print(
"⚠️ Branch created and pushed with conflict markers. "
"A PR will be opened for manual resolution."
)
return applied_cleanly
def create_pull_request(oss_root, branch_name, title, body, dry_run):
@@ -433,6 +426,36 @@ def get_commit_author(commit_hash):
raise
def get_all_co_author_lines(commit_hash, commit_message):
"""
Build a deduplicated list of Co-authored-by lines that includes both
the primary commit author and any Co-authored-by trailers already
present in the commit message.
Returns a list of unique "Co-authored-by: Name <email>" strings.
"""
seen = set()
co_author_lines = []
def _add(name, email):
key = (name.strip(), email.strip().lower())
if key not in seen:
seen.add(key)
co_author_lines.append(f"Co-authored-by: {name.strip()} <{email.strip()}>")
# 1. Primary author of the commit
author_name, author_email = get_commit_author(commit_hash)
_add(author_name, author_email)
# 2. Existing Co-authored-by trailers in the commit message
for line in commit_message.splitlines():
m = re.match(r"^\s*Co-authored-by:\s*(.+?)\s*<([^>]+)>", line, re.IGNORECASE)
if m:
_add(m.group(1), m.group(2))
return co_author_lines
def main():
parser = argparse.ArgumentParser(
description="Copy a commit from the private repo to OSS and open a PR."
@@ -487,33 +510,67 @@ def main():
# 2. Get the OSS repo
oss_root, temp_dir = get_oss_repo(args.dry_run)
# 3. Get original commit author for the co-author line
author_name, author_email = get_commit_author(commit_hash)
# 3. Find the latest OSS commit that was synced into sglang-private.
# This is the correct base for our patch, since the private repo's
# code is based on this sync point.
base_oss_commit = find_latest_oss_sync_commit()
if base_oss_commit:
print(f"ℹ️ Will branch from OSS commit {base_oss_commit}")
else:
print(
"⚠️ Could not determine latest OSS sync commit. "
"Falling back to OSS main HEAD."
)
# 4. Prepare content for the commit and PR based on changed files
# 4. Get all co-author lines (primary author + trailers from commit message)
co_author_lines = get_all_co_author_lines(commit_hash, original_commit_message)
authors_display = "\n".join(co_author_lines)
# 5. Prepare content for the commit and PR based on changed files
file_list_str = "\n".join([f"- {f}" for f in relevant_files])
filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files])
if len(filename_list_str) > 40:
filename_list_str = filename_list_str[:40] + "..."
current_date = datetime.datetime.now().strftime("%Y%m%d")
pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})"
pr_body = (
f"Sync changes from commit `{short_hash}`.\n\n"
f"**Files Changed:**\n{file_list_str}\n\n"
f"Author: {author_name} <{author_email}>"
f"\n\n---\n\n"
f"*This is an automated PR created by scripts/copy_from_oss.py.*"
)
# 5. Create branch, apply patch, and push
# 6. Create branch from the last synced OSS commit, apply patch, and push
branch_name = f"sync-{short_hash}-{current_date}"
co_author_line = f"Co-authored-by: {author_name} <{author_email}>"
commit_message = f"{pr_title}\n\n{co_author_line}"
apply_patch_and_push(
oss_root, patch_file, branch_name, commit_message, args.dry_run
co_authors_block = "\n".join(co_author_lines)
commit_message = f"{pr_title}\n\n{co_authors_block}"
applied_cleanly = apply_patch_and_push(
oss_root,
patch_file,
branch_name,
commit_message,
base_oss_commit,
args.dry_run,
)
# 6. Create Pull Request
# 7. Adjust PR title and body when there are conflicts
if not applied_cleanly:
pr_title = (
f"[Auto Sync][⚠️ Conflicts] Update {filename_list_str} ({current_date})"
)
pr_body_parts = [
f"Sync changes from commit `{short_hash}`.\n",
f"**Files Changed:**\n{file_list_str}\n",
f"**Authors:**\n{authors_display}",
]
if not applied_cleanly:
pr_body_parts.append(
"\n\n⚠️ **This patch had merge conflicts.** "
"The branch contains conflict markers that must be resolved manually. "
"Please check the CI logs for the full patch and conflict details."
)
pr_body_parts.append(
f"\n\n---\n\n"
f"*This is an automated PR created by scripts/copy_to_oss.py.*"
)
pr_body = "\n".join(pr_body_parts)
# 8. Create Pull Request
create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run)
finally:

136
scripts/code_sync/utils.py Normal file
View File

@@ -0,0 +1,136 @@
"""
Shared constants and helpers for code-sync scripts.
"""
import os
import re
import subprocess
from typing import Optional
# --- Configuration Begin ---
# List of folders and files to copy to / from the OSS repo.
# Changes outside these paths will be ignored.
FOLDER_NAMES = [
"3rdparty",
"assets",
"benchmark",
"docker",
"docs",
"examples",
"python/sglang/lang",
"python/sglang/jit_kernel",
"python/sglang/srt",
"python/sglang/test",
"python/sglang/utils.py",
"python/sglang/README.md",
"sgl-kernel",
"test/manual",
"test/registered",
"test/srt",
"test/README.md",
"test/run_suite.py",
"README.md",
]
SYNC_COMMIT_PREFIX = r"\[Automated PR\] Copy OSS code from commit"
# --- Configuration End ---
def write_github_step_summary(content: str) -> None:
"""Append *content* to the GitHub Actions step summary (no-op outside CI)."""
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
if not summary_path:
return
with open(summary_path, "a") as f:
f.write(content)
def get_last_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
"""
Find the most recent sync commit that copied from OSS.
Returns the full private-repo commit hash, or None if not found.
The match is restricted to commits whose **subject** starts with the
sync prefix so that unrelated commits mentioning the phrase in their
body are ignored.
"""
subject_pattern = re.compile("^" + SYNC_COMMIT_PREFIX)
try:
cmd = [
"git",
"log",
"--all",
"--grep",
SYNC_COMMIT_PREFIX,
"--format=%H %s",
]
result = subprocess.run(
cmd,
capture_output=True,
text=True,
check=True,
cwd=repo_root,
).stdout.strip()
for line in result.splitlines():
# Format: "<full_hash> <subject>"
parts = line.split(" ", 1)
if len(parts) != 2:
continue
commit_hash, subject = parts
if subject_pattern.search(subject):
return commit_hash
return None
except subprocess.CalledProcessError as e:
print(f"Error finding last sync commit: {e.stderr}")
return None
def find_latest_oss_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
"""
Search the private repo history for the latest commit whose **subject**
matches "[Automated PR] Copy OSS code from commit {commit_id} on {date}"
and return the embedded **OSS** commit hash.
Returns the short OSS commit hash string, or None if not found.
"""
oss_hash_pattern = re.compile("^" + SYNC_COMMIT_PREFIX + r" ([0-9a-f]+)")
try:
# --grep filters on the full message body, so we request subject-only
# output and validate the pattern against the subject ourselves.
result = subprocess.run(
[
"git",
"log",
"--all",
"--grep",
SYNC_COMMIT_PREFIX,
"--pretty=%s",
],
capture_output=True,
text=True,
check=True,
cwd=repo_root,
)
for subject in result.stdout.strip().splitlines():
m = oss_hash_pattern.search(subject)
if m:
oss_commit = m.group(1)
print(
f"✅ Latest OSS sync commit found: {oss_commit} "
f"(from: {subject})"
)
return oss_commit
print(
"⚠️ No '[Automated PR] Copy OSS code from commit ...' " "found in history."
)
return None
except subprocess.CalledProcessError as e:
print(f"Error searching for OSS sync commits: {e.stderr.strip()}")
return None