From cedb86a950e32af1b22b670213776ccfe15e54b6 Mon Sep 17 00:00:00 2001 From: xrwang8 <68765051+xrwang8@users.noreply.github.com> Date: Wed, 4 Mar 2026 03:59:18 +0800 Subject: [PATCH] Feature:Reserve HTTP server port before model loading to immediately detect port conflicts instead of failing after several minutes of model loading. (#17754) Signed-off-by: xrwang8 --- python/sglang/srt/entrypoints/http_server.py | 150 ++++++++++--------- python/sglang/utils.py | 69 +++++++++ 2 files changed, 148 insertions(+), 71 deletions(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 157e117ec..f185ffcd7 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -173,7 +173,7 @@ from sglang.srt.utils import ( set_uvicorn_logging_configs, ) from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth_level -from sglang.utils import get_exception_traceback +from sglang.utils import _prebind_listening_socket, get_exception_traceback from sglang.version import __version__ logger = logging.getLogger(__name__) @@ -1975,74 +1975,80 @@ def launch_server( 1. The HTTP server, Engine, and TokenizerManager all run in the main process. 2. Inter-process communication is done through IPC (each process uses a different port) via the ZMQ library. """ - # Launch subprocesses - tokenizer_manager, template_manager, scheduler_infos, port_args = ( - _launch_subprocesses( - server_args=server_args, - init_tokenizer_manager_func=init_tokenizer_manager_func, - run_scheduler_process_func=run_scheduler_process_func, - run_detokenizer_process_func=run_detokenizer_process_func, - ) - ) - - # Parse info got from the schedulers - remote_instance_transfer_engine_info = ( - parse_remote_instance_transfer_engine_info_from_scheduler_infos(scheduler_infos) - ) - - # Set global states - set_global_state( - _GlobalState( - tokenizer_manager=tokenizer_manager, - template_manager=template_manager, - scheduler_info=scheduler_infos[0], - remote_instance_transfer_engine_info=remote_instance_transfer_engine_info, - ) - ) - - if server_args.enable_metrics: - add_prometheus_track_response_middleware(app) - - # Pass additional arguments to the lifespan function. - # They will be used for additional initialization setups. - if server_args.tokenizer_worker_num == 1: - # If it is single tokenizer mode, we can pass the arguments by attributes of the app object. - app.is_single_tokenizer_mode = True - app.server_args = server_args - app.warmup_thread_kwargs = dict( - server_args=server_args, - launch_callback=launch_callback, - execute_warmup_func=execute_warmup_func, - ) - - # Add api key authorization - # This is only supported in single tokenizer mode. - # - # Backward compatibility: - # - api_key only: behavior matches legacy (all endpoints require api_key) - # - no keys: legacy had no restriction; ADMIN_FORCE endpoints must still be rejected when - # admin_api_key is not configured. - if ( - server_args.api_key - or server_args.admin_api_key - or app_has_admin_force_endpoints(app) - ): - from sglang.srt.utils.auth import add_api_key_middleware - - add_api_key_middleware( - app, - api_key=server_args.api_key, - admin_api_key=server_args.admin_api_key, - ) - else: - # If it is multi-tokenizer mode, we need to write the arguments to shared memory - # for other worker processes to read. - app.is_single_tokenizer_mode = False - multi_tokenizer_args_shm = write_data_for_multi_tokenizer( - port_args, server_args, scheduler_infos[0] - ) + # Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable. + # This prevents wasting time loading models only to discover port conflicts later. + reserved_socket = _prebind_listening_socket(server_args.host, server_args.port) try: + # Launch subprocesses + tokenizer_manager, template_manager, scheduler_infos, port_args = ( + _launch_subprocesses( + server_args=server_args, + init_tokenizer_manager_func=init_tokenizer_manager_func, + run_scheduler_process_func=run_scheduler_process_func, + run_detokenizer_process_func=run_detokenizer_process_func, + ) + ) + + # Parse info got from the schedulers + remote_instance_transfer_engine_info = ( + parse_remote_instance_transfer_engine_info_from_scheduler_infos( + scheduler_infos + ) + ) + + # Set global states + set_global_state( + _GlobalState( + tokenizer_manager=tokenizer_manager, + template_manager=template_manager, + scheduler_info=scheduler_infos[0], + remote_instance_transfer_engine_info=remote_instance_transfer_engine_info, + ) + ) + + if server_args.enable_metrics: + add_prometheus_track_response_middleware(app) + + # Pass additional arguments to the lifespan function. + # They will be used for additional initialization setups. + if server_args.tokenizer_worker_num == 1: + # If it is single tokenizer mode, we can pass the arguments by attributes of the app object. + app.is_single_tokenizer_mode = True + app.server_args = server_args + app.warmup_thread_kwargs = dict( + server_args=server_args, + launch_callback=launch_callback, + execute_warmup_func=execute_warmup_func, + ) + + # Add api key authorization + # This is only supported in single tokenizer mode. + # + # Backward compatibility: + # - api_key only: behavior matches legacy (all endpoints require api_key) + # - no keys: legacy had no restriction; ADMIN_FORCE endpoints must still be rejected when + # admin_api_key is not configured. + if ( + server_args.api_key + or server_args.admin_api_key + or app_has_admin_force_endpoints(app) + ): + from sglang.srt.utils.auth import add_api_key_middleware + + add_api_key_middleware( + app, + api_key=server_args.api_key, + admin_api_key=server_args.admin_api_key, + ) + else: + # If it is multi-tokenizer mode, we need to write the arguments to shared memory + # for other worker processes to read. + app.is_single_tokenizer_mode = False + multi_tokenizer_args_shm = write_data_for_multi_tokenizer( + port_args, server_args, scheduler_infos[0] + ) + # Update logging configs set_uvicorn_logging_configs(server_args) @@ -2051,8 +2057,7 @@ def launch_server( # Default case, one tokenizer process uvicorn.run( app, - host=server_args.host, - port=server_args.port, + fd=reserved_socket.fileno(), root_path=server_args.fastapi_root_path, log_level=server_args.log_level_http or server_args.log_level, timeout_keep_alive=5, @@ -2071,8 +2076,7 @@ def launch_server( uvicorn.run( "sglang.srt.entrypoints.http_server:app", - host=server_args.host, - port=server_args.port, + fd=reserved_socket.fileno(), root_path=server_args.fastapi_root_path, log_level=server_args.log_level_http or server_args.log_level, timeout_keep_alive=5, @@ -2080,6 +2084,10 @@ def launch_server( workers=server_args.tokenizer_worker_num, ) finally: + # Close the reserved socket after uvicorn exits or on any error + # This ensures the port is released even if initialization fails + reserved_socket.close() + if server_args.tokenizer_worker_num > 1: multi_tokenizer_args_shm.unlink() _global_state.tokenizer_manager.socket_mapping.clear_all_sockets() diff --git a/python/sglang/utils.py b/python/sglang/utils.py index 751d1af33..eb01dde20 100644 --- a/python/sglang/utils.py +++ b/python/sglang/utils.py @@ -409,6 +409,75 @@ def reserve_port(host, start=30000, end=40000): raise RuntimeError("No free port available.") +def _prebind_listening_socket(host: str, port: int) -> socket.socket: + """ + Create and bind a server socket to reserve the port before model loading. + + This prevents port conflicts that could occur if another process (e.g., a + subprocess spawned during model loading) binds to the same port before the + HTTP server starts. By binding early, we ensure the port is available and + reserved for the HTTP server. + + Args: + host: The host address to bind to. + port: The port number to bind to. + + Returns: + A bound socket object. + + Raises: + RuntimeError: If the port is already in use or permission is denied. + """ + import errno + + from sglang.srt.utils.common import is_valid_ipv6_address + + # Determine socket family based on address type + if host and is_valid_ipv6_address(host): + family = socket.AF_INET6 + else: + family = socket.AF_INET + + sock = socket.socket(family=family, type=socket.SOCK_STREAM) + + # Set socket options to allow port reuse + # SO_REUSEADDR: Allows binding to a port in TIME_WAIT state + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # SO_REUSEPORT: Allows multiple processes to bind to the same port + # This is useful for multi-worker mode + if hasattr(socket, "SO_REUSEPORT"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + try: + sock.bind((host or "", port)) + sock.listen(128) # Put socket in listening state for uvicorn + sock.set_inheritable(True) + logger.info( + f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'" + ) + except OSError as e: + sock.close() + if e.errno == errno.EADDRINUSE: + raise RuntimeError( + f"Port {port} is already in use on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'. " + f"Please choose a different port with --port or stop the process using this port. " + f"You can find the process using: lsof -i :{port} or netstat -tlnp | grep {port}" + ) from e + elif e.errno == errno.EACCES: + raise RuntimeError( + f"Permission denied when trying to bind to port {port}. " + f"Ports below 1024 require root/sudo privileges. " + f"Consider using a port >= 1024 (e.g., --port 8000) or run with sudo." + ) from e + else: + raise RuntimeError( + f"Failed to bind to {host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}:{port}: {e}" + ) from e + + return sock + + def release_port(lock_socket): """ Release the reserved port by closing the lock socket.