diff --git a/python/sglang/srt/grpc/grpc_request_manager.py b/python/sglang/srt/grpc/grpc_request_manager.py index dc0c27539..898031af3 100644 --- a/python/sglang/srt/grpc/grpc_request_manager.py +++ b/python/sglang/srt/grpc/grpc_request_manager.py @@ -983,8 +983,8 @@ class GrpcRequestManager: self.event_loop = loop - # We cannot add signal handler when the grpc manager is not in - # the main thread due to the CPython limitation. + # We only add signal handler when the tokenizer manager is in the main thread + # due to the CPython limitation. if threading.current_thread() is threading.main_thread(): signal_handler = GrpcSignalHandler(self) loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler) @@ -992,12 +992,7 @@ class GrpcRequestManager: loop.add_signal_handler( signal.SIGQUIT, signal_handler.running_phase_sigquit_handler ) - else: - logger.warning( - "Signal handler is not added because the grpc request manager is " - "not in the main thread. This disables graceful shutdown of the " - "grpc request manager when SIGTERM is received." - ) + self.asyncio_tasks.add( loop.create_task(print_exception_wrapper(self.sigterm_watchdog)) ) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index e35b5e982..5bbbd5ed0 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1451,6 +1451,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi ) self.event_loop = loop + # We only add signal handler when the tokenizer manager is in the main thread + # due to the CPython limitation. if threading.current_thread() is threading.main_thread(): signal_handler = self.signal_handler_class(self) loop.add_signal_handler(signal.SIGTERM, signal_handler.sigterm_handler) @@ -1458,14 +1460,6 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi loop.add_signal_handler( signal.SIGQUIT, signal_handler.running_phase_sigquit_handler ) - else: - # We cannot add signal handler when the tokenizer manager is not in - # the main thread due to the CPython limitation. - logger.warning( - "Signal handler is not added because the tokenizer manager is " - "not in the main thread. This disables graceful shutdown of the " - "tokenizer manager when SIGTERM is received." - ) self.asyncio_tasks.add( loop.create_task(print_exception_wrapper(self.sigterm_watchdog)) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index edc073e72..d61ddcaf9 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1127,10 +1127,6 @@ def check_pkg_version_at_least(pkg: str, min_version: str) -> bool: def kill_process_tree(parent_pid, include_parent: bool = True, skip_pid: int = None): """Kill the process and all its child processes.""" - # Remove sigchld handler to avoid spammy logs. - if threading.current_thread() is threading.main_thread(): - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - if parent_pid is None: parent_pid = os.getpid() include_parent = False diff --git a/scripts/code_sync/check_commits.py b/scripts/code_sync/check_commits.py index ee08f36a8..33c3a2f5a 100644 --- a/scripts/code_sync/check_commits.py +++ b/scripts/code_sync/check_commits.py @@ -2,13 +2,15 @@ List commits in the private repo that need to be synced to the OSS repo. NOTE: -1. You need to execute this script in the git root folder. +1. This script resolves the git root automatically and can be run anywhere + inside the repo. This script will: 1. Find the most recent sync commit (message starts with "[Automated PR] Copy OSS code from commit"). 2. Scan commits after that point and keep those that touch the configured paths. -3. Print a markdown summary with commit links and write it to GitHub Step Summary. +3. Compare added diff lines in relevant files against OSS main. +4. Print a markdown summary with commit links and write it to GitHub Step Summary. Usage: python3 scripts/code_sync/check_commits.py @@ -18,41 +20,35 @@ import argparse import os import shutil import subprocess -from typing import List, Optional, Tuple +import sys +from dataclasses import dataclass +from typing import Dict, List, Optional, Set, Tuple + +# Allow sibling imports regardless of the working directory. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from utils import ( # noqa: E402 + FOLDER_NAMES, + get_last_sync_commit, + write_github_step_summary, +) # --- Configuration Begin --- -# List of folders and files to copy to the OSS repo. -# Changes outside these paths will be ignored. -folder_names = [ - "3rdparty", - "assets", - "benchmark", - "docker", - "docs", - "examples", - "python/sglang/lang", - "python/sglang/jit_kernel", - "python/sglang/srt", - "python/sglang/test", - "python/sglang/utils.py", - "python/sglang/README.md", - "sgl-kernel", - "test/manual", - "test/registered", - "test/srt", - "test/README.md", - "test/run_suite.py", - "README.md", -] - private_repo = "your-org/sglang-private-repo" -sync_commit_prefix = r"\[Automated PR\] Copy OSS code from commit" +oss_repo_url = "https://github.com/sgl-project/sglang.git" +oss_repo_branch = "main" +default_oss_repo_dir = ".oss_repo" # --- Configuration End --- -def write_github_step_summary(content: str) -> None: - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: - f.write(content) +@dataclass +class CommitInfo: + commit_hash: str + subject: str + commit_date: str + relevant_files: List[str] + synced_lines: int + total_added_lines: int def check_dependencies() -> None: @@ -61,7 +57,23 @@ def check_dependencies() -> None: raise EnvironmentError("git is not installed or not in PATH.") -def get_repo_from_origin() -> str: +def get_repo_root() -> str: + try: + output = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Unable to determine git repo root: {e.stderr or e}") from e + + if not output: + raise RuntimeError("Unable to determine git repo root.") + return os.path.abspath(output) + + +def get_repo_from_origin(repo_root: str) -> str: """Try to infer the repo slug (owner/name) from git remote.origin.url.""" try: url = subprocess.run( @@ -69,6 +81,7 @@ def get_repo_from_origin() -> str: capture_output=True, text=True, check=True, + cwd=repo_root, ).stdout.strip() except subprocess.CalledProcessError: return private_repo @@ -85,29 +98,48 @@ def get_repo_from_origin() -> str: return repo or private_repo -def get_last_sync_commit() -> Optional[str]: - """Find the most recent sync commit that copied from OSS.""" - try: - result = subprocess.run( - [ - "git", - "log", - "-1", - "--grep", - sync_commit_prefix, - "--format=%H", - ], - capture_output=True, - text=True, +def get_default_oss_repo_path(repo_root: str) -> str: + env_path = os.environ.get("OSS_REPO_PATH") + if env_path: + return os.path.abspath(env_path) + return os.path.abspath(os.path.join(repo_root, default_oss_repo_dir)) + + +def ensure_oss_repo(oss_repo_path: str, repo_url: str, branch: str) -> str: + oss_repo_path = os.path.abspath(oss_repo_path) + if os.path.exists(oss_repo_path) and not os.path.isdir(oss_repo_path): + raise RuntimeError(f"OSS repo path is not a directory: {oss_repo_path}") + + if os.path.isdir(os.path.join(oss_repo_path, ".git")): + try: + subprocess.run( + ["git", "-C", oss_repo_path, "rev-parse", "--is-inside-work-tree"], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"OSS repo path exists but is not a git repo: {oss_repo_path}" + ) from e + + subprocess.run( + ["git", "-C", oss_repo_path, "fetch", "origin", branch, "--depth", "1"], check=True, - ).stdout.strip() - return result or None - except subprocess.CalledProcessError as e: - print(f"Error finding last sync commit: {e.stderr}") - return None + ) + return oss_repo_path + + parent_dir = os.path.dirname(oss_repo_path) + if parent_dir and not os.path.isdir(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + subprocess.run( + ["git", "clone", "--depth", "1", "--branch", branch, repo_url, oss_repo_path], + check=True, + ) + return oss_repo_path -def get_commits_since(last_sync_hash: Optional[str]) -> List[str]: +def get_commits_since(repo_root: str, last_sync_hash: Optional[str]) -> List[str]: """Get commit hashes from last sync commit (exclusive) to HEAD.""" try: if last_sync_hash: @@ -115,7 +147,7 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]: else: command = ["git", "rev-list", "HEAD"] result = subprocess.run( - command, capture_output=True, text=True, check=True + command, capture_output=True, text=True, check=True, cwd=repo_root ).stdout.strip() return [line for line in result.split("\n") if line] except subprocess.CalledProcessError as e: @@ -123,13 +155,14 @@ def get_commits_since(last_sync_hash: Optional[str]) -> List[str]: return [] -def get_changed_files(commit_hash: str) -> List[str]: +def get_changed_files(repo_root: str, commit_hash: str) -> List[str]: try: output = subprocess.run( ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash], capture_output=True, text=True, check=True, + cwd=repo_root, ).stdout.strip() return [line for line in output.split("\n") if line] except subprocess.CalledProcessError as e: @@ -147,11 +180,115 @@ def get_relevant_files(changed_files: List[str]) -> List[str]: return [ changed_file for changed_file in changed_files - if any(is_relevant_path(changed_file, path) for path in folder_names) + if any(is_relevant_path(changed_file, path) for path in FOLDER_NAMES) ] -def get_commit_summary(commit_hash: str) -> Tuple[str, str]: +def get_added_lines_by_file( + repo_root: str, commit_hash: str, relevant_files: List[str] +) -> Dict[str, List[str]]: + if not relevant_files: + return {} + + command = [ + "git", + "show", + "--no-color", + "--unified=0", + "--format=", + commit_hash, + "--", + ] + relevant_files + try: + output = subprocess.run( + command, capture_output=True, text=True, check=True, cwd=repo_root + ).stdout + except subprocess.CalledProcessError as e: + print(f"Error getting diff for {commit_hash}: {e.stderr}") + return {} + + added_lines: Dict[str, List[str]] = {path: [] for path in relevant_files} + relevant_set = set(relevant_files) + current_file: Optional[str] = None + for line in output.splitlines(): + if line.startswith("diff --git "): + current_file = None + continue + if line.startswith("+++ "): + file_path = None + if line.startswith("+++ b/"): + file_path = line[6:] + else: + candidate = line[4:] + if candidate == "/dev/null": + file_path = None + elif candidate.startswith("b/") or candidate.startswith("a/"): + file_path = candidate[2:] + else: + file_path = candidate + + if file_path in relevant_set: + current_file = file_path + else: + current_file = None + continue + + if current_file and line.startswith("+") and not line.startswith("+++ "): + added_lines[current_file].append(line[1:]) + + return added_lines + + +def get_oss_file_lines( + oss_repo_path: str, + oss_ref: str, + file_path: str, + cache: Dict[str, Optional[Set[str]]], +) -> Optional[Set[str]]: + if file_path in cache: + return cache[file_path] + try: + output = subprocess.run( + ["git", "-C", oss_repo_path, "show", f"{oss_ref}:{file_path}"], + capture_output=True, + text=True, + errors="replace", + check=True, + ).stdout + except subprocess.CalledProcessError: + cache[file_path] = None + return None + + lines = output.splitlines() + line_set = set(lines) + cache[file_path] = line_set + return line_set + + +def count_synced_lines( + added_lines_by_file: Dict[str, List[str]], + oss_repo_path: str, + oss_ref: str, + oss_file_cache: Dict[str, Optional[Set[str]]], +) -> Tuple[int, int]: + total_added_lines = 0 + synced_lines = 0 + for file_path, lines in added_lines_by_file.items(): + total_added_lines += len(lines) + if not lines: + continue + oss_lines = get_oss_file_lines( + oss_repo_path, oss_ref, file_path, oss_file_cache + ) + if not oss_lines: + continue + for line in lines: + if line in oss_lines: + synced_lines += 1 + return synced_lines, total_added_lines + + +def get_commit_summary(repo_root: str, commit_hash: str) -> Tuple[str, str]: """Return (subject, date) for a commit.""" try: output = subprocess.run( @@ -159,6 +296,7 @@ def get_commit_summary(commit_hash: str) -> Tuple[str, str]: capture_output=True, text=True, check=True, + cwd=repo_root, ).stdout.strip() subject, commit_date = output.split("\x00", 1) except subprocess.CalledProcessError as e: @@ -195,13 +333,20 @@ def format_commit_block( commit_hash: str, commit_date: str, relevant_files: List[str], + synced_lines: int, + total_added_lines: int, ) -> str: short_hash = commit_hash[:9] commit_url = f"https://github.com/{repo}/commit/{commit_hash}" files_str = format_files_list(relevant_files) if relevant_files else "- None" + status_icon = "✅" if synced_lines == total_added_lines else "❌" + status_line = ( + f"status: {status_icon} {synced_lines}/{total_added_lines} lines synced" + ) return "\n".join( [ f"#### {subject}", + status_line, f"date: {commit_date}", "files to sync:", files_str, @@ -215,7 +360,7 @@ def format_commit_block( def format_output( repo: str, last_sync: Optional[Tuple[str, str, str]], - commits: List[Tuple[str, str, str, List[str]]], + commits: List[CommitInfo], ) -> str: lines: List[str] = [] if last_sync: @@ -229,9 +374,17 @@ def format_output( lines.append("No commits need to be synced.") return "\n".join(lines) + "\n" - for commit_hash, subject, commit_date, relevant_files in commits: + for commit in commits: lines.append( - format_commit_block(repo, subject, commit_hash, commit_date, relevant_files) + format_commit_block( + repo, + commit.subject, + commit.commit_hash, + commit.commit_date, + commit.relevant_files, + commit.synced_lines, + commit.total_added_lines, + ) ) return "\n".join(lines) @@ -247,30 +400,78 @@ def main() -> None: default=0, help="Limit number of commits printed (0 means no limit).", ) + parser.add_argument( + "--oss-repo-path", + default=None, + help="Path to OSS repo clone (default: $OSS_REPO_PATH or .oss_repo).", + ) + parser.add_argument( + "--oss-repo-url", + default=oss_repo_url, + help="OSS repo URL (default: https://github.com/sgl-project/sglang.git).", + ) + parser.add_argument( + "--oss-branch", + default=oss_repo_branch, + help="OSS repo branch to check (default: main).", + ) args = parser.parse_args() check_dependencies() + repo_root = get_repo_root() + oss_repo_path = ( + os.path.abspath(args.oss_repo_path) + if args.oss_repo_path + else get_default_oss_repo_path(repo_root) + ) - repo = get_repo_from_origin() - last_sync_hash = get_last_sync_commit() + repo = get_repo_from_origin(repo_root) + last_sync_hash = get_last_sync_commit(repo_root) last_sync_block = None if last_sync_hash: - last_sync_subject, last_sync_date = get_commit_summary(last_sync_hash) + last_sync_subject, last_sync_date = get_commit_summary( + repo_root, last_sync_hash + ) last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date) - commits = get_commits_since(last_sync_hash) + commits = get_commits_since(repo_root, last_sync_hash) if args.limit > 0: commits = commits[: args.limit] - relevant_commits: List[Tuple[str, str, str, List[str]]] = [] + relevant_commit_inputs: List[Tuple[str, List[str]]] = [] for commit_hash in commits: - changed_files = get_changed_files(commit_hash) + changed_files = get_changed_files(repo_root, commit_hash) if not changed_files: continue relevant_files = get_relevant_files(changed_files) if relevant_files: - subject, commit_date = get_commit_summary(commit_hash) - relevant_commits.append((commit_hash, subject, commit_date, relevant_files)) + relevant_commit_inputs.append((commit_hash, relevant_files)) + + relevant_commits: List[CommitInfo] = [] + if relevant_commit_inputs: + oss_repo_path = ensure_oss_repo( + oss_repo_path, args.oss_repo_url, args.oss_branch + ) + oss_ref = f"origin/{args.oss_branch}" + oss_file_cache: Dict[str, Optional[Set[str]]] = {} + for commit_hash, relevant_files in relevant_commit_inputs: + subject, commit_date = get_commit_summary(repo_root, commit_hash) + added_lines_by_file = get_added_lines_by_file( + repo_root, commit_hash, relevant_files + ) + synced_lines, total_added_lines = count_synced_lines( + added_lines_by_file, oss_repo_path, oss_ref, oss_file_cache + ) + relevant_commits.append( + CommitInfo( + commit_hash=commit_hash, + subject=subject, + commit_date=commit_date, + relevant_files=relevant_files, + synced_lines=synced_lines, + total_added_lines=total_added_lines, + ) + ) output = format_output(repo, last_sync_block, relevant_commits) print(output) diff --git a/scripts/code_sync/copy_from_oss.py b/scripts/code_sync/copy_from_oss.py index a9c5a0bdd..6af3c51a0 100644 --- a/scripts/code_sync/copy_from_oss.py +++ b/scripts/code_sync/copy_from_oss.py @@ -31,45 +31,19 @@ import datetime import os import shutil import subprocess +import sys import tempfile -# --- Configuration Begin --- -# List of folders and files to copy from the OSS repo. -# Changes outside these paths will be ignored. -folder_names = [ - "3rdparty", - "assets", - "benchmark", - "docker", - "docs", - "examples", - "python/sglang/lang", - "python/sglang/jit_kernel", - "python/sglang/srt", - "python/sglang/test", - "python/sglang/utils.py", - "python/sglang/README.md", - "sgl-kernel", - "test/manual", - "test/registered", - "test/srt", - "test/README.md", - "test/run_suite.py", - "README.md", -] +# Allow sibling imports regardless of the working directory. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from utils import FOLDER_NAMES, write_github_step_summary # noqa: E402 + +# --- Configuration Begin --- private_repo = "your-org/sglang-private-repo" # --- Configuration End --- -def write_github_step_summary(content): - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: - f.write(content) - - def check_dependencies(): """Check for required command-line tools.""" if not shutil.which("git"): @@ -145,10 +119,10 @@ def get_source_folder(args): return oss_root, temp_dir, commit_hash -def sync_directories(oss_root, folder_names, dry_run): +def sync_directories(oss_root, sync_paths, dry_run): """Sync specified directories from oss_root to current working directory.""" rsync_commands = [] - for folder_name in folder_names: + for folder_name in sync_paths: target_name = f"{oss_root}/{folder_name}" src_name = "./" + "/".join(folder_name.split("/")[:-1]) cmd = f"rsync -r --delete {target_name} {src_name}" @@ -259,7 +233,7 @@ def main(): try: # Sync directories - sync_directories(oss_root, folder_names, args.dry_run) + sync_directories(oss_root, FOLDER_NAMES, args.dry_run) # Check for changes and create PR if necessary if not check_for_changes(): diff --git a/scripts/code_sync/copy_to_oss.py b/scripts/code_sync/copy_to_oss.py index 82a6469f8..d87a81dd7 100644 --- a/scripts/code_sync/copy_to_oss.py +++ b/scripts/code_sync/copy_to_oss.py @@ -29,44 +29,20 @@ python3 scripts/copy_to_oss.py --commit --dry-run import argparse import datetime import os +import re import shutil import subprocess +import sys import tempfile -# --- Configuration Begin --- -# List of folders and files to copy to the OSS repo. -# Changes outside these paths will be ignored. -folder_names = [ - "3rdparty", - "assets", - "benchmark", - "docker", - "docs", - "examples", - "python/sglang/lang", - "python/sglang/jit_kernel", - "python/sglang/srt", - "python/sglang/test", - "python/sglang/utils.py", - "python/sglang/README.md", - "sgl-kernel", - "test/manual", - "test/registered", - "test/srt", - "test/README.md", - "test/run_suite.py", - "README.md", -] +# Allow sibling imports regardless of the working directory. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -# --- Configuration End --- - - -def write_github_step_summary(content): - if not os.environ.get("GITHUB_STEP_SUMMARY"): - return - - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: - f.write(content) +from utils import ( # noqa: E402 + FOLDER_NAMES, + find_latest_oss_sync_commit, + write_github_step_summary, +) def get_commit_info(commit_ref): @@ -131,7 +107,7 @@ def create_filtered_patch(commit_hash, dry_run): # Filter the list of files relevant_files = [ - f for f in changed_files if any(f.startswith(path) for path in folder_names) + f for f in changed_files if any(f.startswith(path) for path in FOLDER_NAMES) ] if not relevant_files: @@ -190,7 +166,7 @@ def get_oss_repo(dry_run): print(f"\nCreated temporary directory for OSS repo: {temp_dir}") repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git" - command = ["git", "clone", "--branch", "main", repo_url, oss_root] + command = ["git", "clone", repo_url, oss_root] print(f"Run: {' '.join(command)}") if not dry_run: @@ -209,8 +185,9 @@ def _apply_patch(patch_file, dry_run): """ Try to apply a patch, falling back to --3way merge if a clean apply fails. - Returns True if the patch was applied successfully (clean or via --3way), - or raises an exception with full diagnostics if both methods fail. + Returns True if the patch was applied cleanly. + Returns False if conflicts were encountered (changes are still staged + with conflict markers so a PR can be created for manual resolution). """ # --- Attempt 1: clean git apply --- apply_cmd = ["git", "apply", patch_file] @@ -235,8 +212,8 @@ def _apply_patch(patch_file, dry_run): print("✅ Patch applied via --3way merge (no conflicts).") return True - # --- Both attempts failed — gather diagnostics --- - print(f"❌ --3way merge also failed:\n{result_3way.stderr.strip()}\n") + # --- --3way left conflict markers in the working tree --- + print(f"⚠️ --3way merge had conflicts:\n{result_3way.stderr.strip()}\n") # Show which hunks conflict check_cmd = ["git", "apply", "--check", "--verbose", patch_file] @@ -272,7 +249,7 @@ def _apply_patch(patch_file, dry_run): # Write a rich summary to the GitHub Actions step summary summary_lines = [ - "\n## ❌ Patch could not be applied automatically\n", + "\n## ⚠️ Patch had conflicts — PR created for manual resolution\n", "### Conflict details\n", f"```\n{conflict_details}\n```\n", ] @@ -287,18 +264,22 @@ def _apply_patch(patch_file, dry_run): ) write_github_step_summary("".join(summary_lines)) - # Reset the working tree so the finally-block cleanup is safe - subprocess.run(["git", "checkout", "."], capture_output=True, text=True) - - raise RuntimeError( - "Patch could not be applied. See the CI log and GitHub Actions " - "step summary for the full patch and conflict details." - ) + return False -def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run): +def apply_patch_and_push( + oss_root, patch_file, branch_name, commit_message, base_oss_commit, dry_run +): """ - In the OSS repo, create a branch, apply the patch, commit, and push. + In the OSS repo, create a branch from base_oss_commit, apply the patch, + commit, and push. + + Args: + base_oss_commit: The OSS commit hash to branch from (the last sync + point). If None, the current HEAD (main) is used. + + Returns True if the patch applied cleanly, False if there were conflicts + (the conflicted state is still committed and pushed so a PR can be opened). """ print("\nApplying patch and pushing to OSS repo...") @@ -306,15 +287,19 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_ if not dry_run: os.chdir(oss_root) + applied_cleanly = True try: - # Create a new branch - checkout_cmd = ["git", "checkout", "-b", branch_name] + # Check out a new branch from the base OSS commit + if base_oss_commit: + checkout_cmd = ["git", "checkout", "-b", branch_name, base_oss_commit] + else: + checkout_cmd = ["git", "checkout", "-b", branch_name] print(f"Run: {' '.join(checkout_cmd)}") if not dry_run: subprocess.run(checkout_cmd, check=True, capture_output=True, text=True) # Apply the patch (with --3way fallback and diagnostics) - _apply_patch(patch_file, dry_run) + applied_cleanly = _apply_patch(patch_file, dry_run) # Configure git user and stage changes post_apply_commands = [ @@ -359,7 +344,15 @@ def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_ if not dry_run: os.chdir(original_cwd) - print("✅ Branch created, patch applied, and pushed successfully.") + if applied_cleanly: + print("✅ Branch created, patch applied cleanly, and pushed successfully.") + else: + print( + "⚠️ Branch created and pushed with conflict markers. " + "A PR will be opened for manual resolution." + ) + + return applied_cleanly def create_pull_request(oss_root, branch_name, title, body, dry_run): @@ -433,6 +426,36 @@ def get_commit_author(commit_hash): raise +def get_all_co_author_lines(commit_hash, commit_message): + """ + Build a deduplicated list of Co-authored-by lines that includes both + the primary commit author and any Co-authored-by trailers already + present in the commit message. + + Returns a list of unique "Co-authored-by: Name " strings. + """ + seen = set() + co_author_lines = [] + + def _add(name, email): + key = (name.strip(), email.strip().lower()) + if key not in seen: + seen.add(key) + co_author_lines.append(f"Co-authored-by: {name.strip()} <{email.strip()}>") + + # 1. Primary author of the commit + author_name, author_email = get_commit_author(commit_hash) + _add(author_name, author_email) + + # 2. Existing Co-authored-by trailers in the commit message + for line in commit_message.splitlines(): + m = re.match(r"^\s*Co-authored-by:\s*(.+?)\s*<([^>]+)>", line, re.IGNORECASE) + if m: + _add(m.group(1), m.group(2)) + + return co_author_lines + + def main(): parser = argparse.ArgumentParser( description="Copy a commit from the private repo to OSS and open a PR." @@ -487,33 +510,67 @@ def main(): # 2. Get the OSS repo oss_root, temp_dir = get_oss_repo(args.dry_run) - # 3. Get original commit author for the co-author line - author_name, author_email = get_commit_author(commit_hash) + # 3. Find the latest OSS commit that was synced into sglang-private. + # This is the correct base for our patch, since the private repo's + # code is based on this sync point. + base_oss_commit = find_latest_oss_sync_commit() + if base_oss_commit: + print(f"ℹ️ Will branch from OSS commit {base_oss_commit}") + else: + print( + "⚠️ Could not determine latest OSS sync commit. " + "Falling back to OSS main HEAD." + ) - # 4. Prepare content for the commit and PR based on changed files + # 4. Get all co-author lines (primary author + trailers from commit message) + co_author_lines = get_all_co_author_lines(commit_hash, original_commit_message) + authors_display = "\n".join(co_author_lines) + + # 5. Prepare content for the commit and PR based on changed files file_list_str = "\n".join([f"- {f}" for f in relevant_files]) filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files]) if len(filename_list_str) > 40: filename_list_str = filename_list_str[:40] + "..." current_date = datetime.datetime.now().strftime("%Y%m%d") pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})" - pr_body = ( - f"Sync changes from commit `{short_hash}`.\n\n" - f"**Files Changed:**\n{file_list_str}\n\n" - f"Author: {author_name} <{author_email}>" - f"\n\n---\n\n" - f"*This is an automated PR created by scripts/copy_from_oss.py.*" - ) - # 5. Create branch, apply patch, and push + # 6. Create branch from the last synced OSS commit, apply patch, and push branch_name = f"sync-{short_hash}-{current_date}" - co_author_line = f"Co-authored-by: {author_name} <{author_email}>" - commit_message = f"{pr_title}\n\n{co_author_line}" - apply_patch_and_push( - oss_root, patch_file, branch_name, commit_message, args.dry_run + co_authors_block = "\n".join(co_author_lines) + commit_message = f"{pr_title}\n\n{co_authors_block}" + applied_cleanly = apply_patch_and_push( + oss_root, + patch_file, + branch_name, + commit_message, + base_oss_commit, + args.dry_run, ) - # 6. Create Pull Request + # 7. Adjust PR title and body when there are conflicts + if not applied_cleanly: + pr_title = ( + f"[Auto Sync][⚠️ Conflicts] Update {filename_list_str} ({current_date})" + ) + + pr_body_parts = [ + f"Sync changes from commit `{short_hash}`.\n", + f"**Files Changed:**\n{file_list_str}\n", + f"**Authors:**\n{authors_display}", + ] + if not applied_cleanly: + pr_body_parts.append( + "\n\n⚠️ **This patch had merge conflicts.** " + "The branch contains conflict markers that must be resolved manually. " + "Please check the CI logs for the full patch and conflict details." + ) + pr_body_parts.append( + f"\n\n---\n\n" + f"*This is an automated PR created by scripts/copy_to_oss.py.*" + ) + pr_body = "\n".join(pr_body_parts) + + # 8. Create Pull Request create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run) finally: diff --git a/scripts/code_sync/utils.py b/scripts/code_sync/utils.py new file mode 100644 index 000000000..410417f20 --- /dev/null +++ b/scripts/code_sync/utils.py @@ -0,0 +1,136 @@ +""" +Shared constants and helpers for code-sync scripts. +""" + +import os +import re +import subprocess +from typing import Optional + +# --- Configuration Begin --- +# List of folders and files to copy to / from the OSS repo. +# Changes outside these paths will be ignored. +FOLDER_NAMES = [ + "3rdparty", + "assets", + "benchmark", + "docker", + "docs", + "examples", + "python/sglang/lang", + "python/sglang/jit_kernel", + "python/sglang/srt", + "python/sglang/test", + "python/sglang/utils.py", + "python/sglang/README.md", + "sgl-kernel", + "test/manual", + "test/registered", + "test/srt", + "test/README.md", + "test/run_suite.py", + "README.md", +] + +SYNC_COMMIT_PREFIX = r"\[Automated PR\] Copy OSS code from commit" +# --- Configuration End --- + + +def write_github_step_summary(content: str) -> None: + """Append *content* to the GitHub Actions step summary (no-op outside CI).""" + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + with open(summary_path, "a") as f: + f.write(content) + + +def get_last_sync_commit(repo_root: Optional[str] = None) -> Optional[str]: + """ + Find the most recent sync commit that copied from OSS. + + Returns the full private-repo commit hash, or None if not found. + The match is restricted to commits whose **subject** starts with the + sync prefix so that unrelated commits mentioning the phrase in their + body are ignored. + """ + subject_pattern = re.compile("^" + SYNC_COMMIT_PREFIX) + + try: + cmd = [ + "git", + "log", + "--all", + "--grep", + SYNC_COMMIT_PREFIX, + "--format=%H %s", + ] + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + cwd=repo_root, + ).stdout.strip() + + for line in result.splitlines(): + # Format: " " + parts = line.split(" ", 1) + if len(parts) != 2: + continue + commit_hash, subject = parts + if subject_pattern.search(subject): + return commit_hash + + return None + except subprocess.CalledProcessError as e: + print(f"Error finding last sync commit: {e.stderr}") + return None + + +def find_latest_oss_sync_commit(repo_root: Optional[str] = None) -> Optional[str]: + """ + Search the private repo history for the latest commit whose **subject** + matches "[Automated PR] Copy OSS code from commit {commit_id} on {date}" + and return the embedded **OSS** commit hash. + + Returns the short OSS commit hash string, or None if not found. + """ + oss_hash_pattern = re.compile("^" + SYNC_COMMIT_PREFIX + r" ([0-9a-f]+)") + + try: + # --grep filters on the full message body, so we request subject-only + # output and validate the pattern against the subject ourselves. + result = subprocess.run( + [ + "git", + "log", + "--all", + "--grep", + SYNC_COMMIT_PREFIX, + "--pretty=%s", + ], + capture_output=True, + text=True, + check=True, + cwd=repo_root, + ) + + for subject in result.stdout.strip().splitlines(): + m = oss_hash_pattern.search(subject) + if m: + oss_commit = m.group(1) + print( + f"✅ Latest OSS sync commit found: {oss_commit} " + f"(from: {subject})" + ) + return oss_commit + + print( + "⚠️ No '[Automated PR] Copy OSS code from commit ...' " "found in history." + ) + return None + + except subprocess.CalledProcessError as e: + print(f"Error searching for OSS sync commits: {e.stderr.strip()}") + return None