chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Feature:Reserve HTTP server port before model loading to immediately detect port conflicts instead of failing after several minutes of model loading. (#17754)

Browse Source 
Signed-off-by: xrwang8 <xrwang8@gmail.com>
This commit is contained in:
xrwang8
2026-03-04 03:59:18 +08:00
committed by GitHub
parent 2e1b9e2547
commit cedb86a950
2 changed files with 148 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
python/sglang/srt/entrypoints/http_server.py
View File

@@ -173,7 +173,7 @@ from sglang.srt.utils import (
set_uvicorn_logging_configs,
)
from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth_level
from sglang.utils import get_exception_traceback
from sglang.utils import _prebind_listening_socket, get_exception_traceback
from sglang.version import __version__
logger = logging.getLogger(__name__)
@@ -1975,74 +1975,80 @@ def launch_server(
1. The HTTP server, Engine, and TokenizerManager all run in the main process.
2. Inter-process communication is done through IPC (each process uses a different port) via the ZMQ library.
"""
# Launch subprocesses
tokenizer_manager, template_manager, scheduler_infos, port_args = (
_launch_subprocesses(
server_args=server_args,
init_tokenizer_manager_func=init_tokenizer_manager_func,
run_scheduler_process_func=run_scheduler_process_func,
run_detokenizer_process_func=run_detokenizer_process_func,
)
)
# Parse info got from the schedulers
remote_instance_transfer_engine_info = (
parse_remote_instance_transfer_engine_info_from_scheduler_infos(scheduler_infos)
)
# Set global states
set_global_state(
_GlobalState(
tokenizer_manager=tokenizer_manager,
template_manager=template_manager,
scheduler_info=scheduler_infos[0],
remote_instance_transfer_engine_info=remote_instance_transfer_engine_info,
)
)
if server_args.enable_metrics:
add_prometheus_track_response_middleware(app)
# Pass additional arguments to the lifespan function.
# They will be used for additional initialization setups.
if server_args.tokenizer_worker_num == 1:
# If it is single tokenizer mode, we can pass the arguments by attributes of the app object.
app.is_single_tokenizer_mode = True
app.server_args = server_args
app.warmup_thread_kwargs = dict(
server_args=server_args,
launch_callback=launch_callback,
execute_warmup_func=execute_warmup_func,
)
# Add api key authorization
# This is only supported in single tokenizer mode.
#
# Backward compatibility:
# - api_key only: behavior matches legacy (all endpoints require api_key)
# - no keys: legacy had no restriction; ADMIN_FORCE endpoints must still be rejected when
# admin_api_key is not configured.
if (
server_args.api_key
or server_args.admin_api_key
or app_has_admin_force_endpoints(app)
):
from sglang.srt.utils.auth import add_api_key_middleware
add_api_key_middleware(
app,
api_key=server_args.api_key,
admin_api_key=server_args.admin_api_key,
)
else:
# If it is multi-tokenizer mode, we need to write the arguments to shared memory
# for other worker processes to read.
app.is_single_tokenizer_mode = False
multi_tokenizer_args_shm = write_data_for_multi_tokenizer(
port_args, server_args, scheduler_infos[0]
)
# Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable.
# This prevents wasting time loading models only to discover port conflicts later.
reserved_socket = _prebind_listening_socket(server_args.host, server_args.port)
try:
# Launch subprocesses
tokenizer_manager, template_manager, scheduler_infos, port_args = (
_launch_subprocesses(
server_args=server_args,
init_tokenizer_manager_func=init_tokenizer_manager_func,
run_scheduler_process_func=run_scheduler_process_func,
run_detokenizer_process_func=run_detokenizer_process_func,
)
)
# Parse info got from the schedulers
remote_instance_transfer_engine_info = (
parse_remote_instance_transfer_engine_info_from_scheduler_infos(
scheduler_infos
)
)
# Set global states
set_global_state(
_GlobalState(
tokenizer_manager=tokenizer_manager,
template_manager=template_manager,
scheduler_info=scheduler_infos[0],
remote_instance_transfer_engine_info=remote_instance_transfer_engine_info,
)
)
if server_args.enable_metrics:
add_prometheus_track_response_middleware(app)
# Pass additional arguments to the lifespan function.
# They will be used for additional initialization setups.
if server_args.tokenizer_worker_num == 1:
# If it is single tokenizer mode, we can pass the arguments by attributes of the app object.
app.is_single_tokenizer_mode = True
app.server_args = server_args
app.warmup_thread_kwargs = dict(
server_args=server_args,
launch_callback=launch_callback,
execute_warmup_func=execute_warmup_func,
)
# Add api key authorization
# This is only supported in single tokenizer mode.
#
# Backward compatibility:
# - api_key only: behavior matches legacy (all endpoints require api_key)
# - no keys: legacy had no restriction; ADMIN_FORCE endpoints must still be rejected when
# admin_api_key is not configured.
if (
server_args.api_key
or server_args.admin_api_key
or app_has_admin_force_endpoints(app)
):
from sglang.srt.utils.auth import add_api_key_middleware
add_api_key_middleware(
app,
api_key=server_args.api_key,
admin_api_key=server_args.admin_api_key,
)
else:
# If it is multi-tokenizer mode, we need to write the arguments to shared memory
# for other worker processes to read.
app.is_single_tokenizer_mode = False
multi_tokenizer_args_shm = write_data_for_multi_tokenizer(
port_args, server_args, scheduler_infos[0]
)
# Update logging configs
set_uvicorn_logging_configs(server_args)
@@ -2051,8 +2057,7 @@ def launch_server(
# Default case, one tokenizer process
uvicorn.run(
app,
host=server_args.host,
port=server_args.port,
fd=reserved_socket.fileno(),
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=5,
@@ -2071,8 +2076,7 @@ def launch_server(
uvicorn.run(
"sglang.srt.entrypoints.http_server:app",
host=server_args.host,
port=server_args.port,
fd=reserved_socket.fileno(),
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=5,
@@ -2080,6 +2084,10 @@ def launch_server(
workers=server_args.tokenizer_worker_num,
)
finally:
# Close the reserved socket after uvicorn exits or on any error
# This ensures the port is released even if initialization fails
reserved_socket.close()
if server_args.tokenizer_worker_num > 1:
multi_tokenizer_args_shm.unlink()
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()

69
python/sglang/utils.py
View File

@@ -409,6 +409,75 @@ def reserve_port(host, start=30000, end=40000):
raise RuntimeError("No free port available.")
def _prebind_listening_socket(host: str, port: int) -> socket.socket:
"""
Create and bind a server socket to reserve the port before model loading.
This prevents port conflicts that could occur if another process (e.g., a
subprocess spawned during model loading) binds to the same port before the
HTTP server starts. By binding early, we ensure the port is available and
reserved for the HTTP server.
Args:
host: The host address to bind to.
port: The port number to bind to.
Returns:
A bound socket object.
Raises:
RuntimeError: If the port is already in use or permission is denied.
"""
import errno
from sglang.srt.utils.common import is_valid_ipv6_address
# Determine socket family based on address type
if host and is_valid_ipv6_address(host):
family = socket.AF_INET6
else:
family = socket.AF_INET
sock = socket.socket(family=family, type=socket.SOCK_STREAM)
# Set socket options to allow port reuse
# SO_REUSEADDR: Allows binding to a port in TIME_WAIT state
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# SO_REUSEPORT: Allows multiple processes to bind to the same port
# This is useful for multi-worker mode
if hasattr(socket, "SO_REUSEPORT"):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
try:
sock.bind((host or "", port))
sock.listen(128) # Put socket in listening state for uvicorn
sock.set_inheritable(True)
logger.info(
f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'"
)
except OSError as e:
sock.close()
if e.errno == errno.EADDRINUSE:
raise RuntimeError(
f"Port {port} is already in use on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'. "
f"Please choose a different port with --port or stop the process using this port. "
f"You can find the process using: lsof -i :{port} or netstat -tlnp | grep {port}"
) from e
elif e.errno == errno.EACCES:
raise RuntimeError(
f"Permission denied when trying to bind to port {port}. "
f"Ports below 1024 require root/sudo privileges. "
f"Consider using a port >= 1024 (e.g., --port 8000) or run with sudo."
) from e
else:
raise RuntimeError(
f"Failed to bind to {host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}:{port}: {e}"
) from e
return sock
def release_port(lock_socket):
"""
Release the reserved port by closing the lock socket.