chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add SSL/TLS support for HTTP and gRPC servers (#18973)

Browse Source 
Co-authored-by: guys@spotify.com
This commit is contained in:
Kangyan-Zhou
2026-03-04 19:27:16 -08:00
committed by GitHub
parent 9c11a7ae40
commit 198381d9ce
9 changed files with 770 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
python/pyproject.toml
View File

@@ -73,6 +73,7 @@ dependencies = [
"transformers==4.57.1",
"uvicorn",
"uvloop",
"watchfiles",
"xgrammar==0.1.27",
"smg-grpc-proto>=0.4.1",

11
python/sglang/multimodal_gen/runtime/server_args.py
View File

@@ -929,10 +929,15 @@ class ServerArgs:
return parser
def url(self):
if is_valid_ipv6_address(self.host):
return f"http://[{self.host}]:{self.port}"
host = self.host
if not host or host == "0.0.0.0":
host = "127.0.0.1"
elif host == "::":
host = "::1"
if is_valid_ipv6_address(host):
return f"http://[{host}]:{self.port}"
else:
return f"http://{self.host}:{self.port}"
return f"http://{host}:{self.port}"
@property
def scheduler_endpoint(self):

126
python/sglang/srt/entrypoints/grpc_server.py
View File

@@ -1050,10 +1050,132 @@ async def serve_grpc(
# Start server
listen_addr = f"{server_args.host}:{server_args.port}"
server.add_insecure_port(listen_addr)
if server_args.ssl_certfile and server_args.ssl_keyfile:
if server_args.ssl_keyfile_password:
raise ValueError(
"gRPC mode does not support encrypted SSL key files "
"(--ssl-keyfile-password). Please provide an unencrypted key "
"file when using --grpc-mode."
)
def _read_ssl_file(filepath: str, description: str) -> bytes:
try:
with open(filepath, "rb") as f:
return f.read()
except OSError as e:
raise ValueError(
f"Failed to read {description} '{filepath}': {e}"
) from e
private_key = _read_ssl_file(server_args.ssl_keyfile, "SSL key file")
certificate_chain = _read_ssl_file(
server_args.ssl_certfile, "SSL certificate file"
)
root_certificates = None
if server_args.ssl_ca_certs:
root_certificates = _read_ssl_file(
server_args.ssl_ca_certs, "SSL CA certificates file"
)
if server_args.enable_ssl_refresh:
# Use dynamic credentials so gRPC re-reads certs on each
# new connection via the fetcher callback.
_cert_mtime = os.path.getmtime(server_args.ssl_certfile)
_key_mtime = os.path.getmtime(server_args.ssl_keyfile)
_ca_mtime = (
os.path.getmtime(server_args.ssl_ca_certs)
if server_args.ssl_ca_certs
else None
)
def _cert_config_fetcher():
nonlocal _cert_mtime, _key_mtime, _ca_mtime
try:
new_cert_mt = os.path.getmtime(server_args.ssl_certfile)
new_key_mt = os.path.getmtime(server_args.ssl_keyfile)
new_ca_mt = (
os.path.getmtime(server_args.ssl_ca_certs)
if server_args.ssl_ca_certs
else None
)
if (
new_cert_mt == _cert_mtime
and new_key_mt == _key_mtime
and new_ca_mt == _ca_mtime
):
return None # No change
new_key = _read_ssl_file(server_args.ssl_keyfile, "SSL key file")
new_cert = _read_ssl_file(
server_args.ssl_certfile, "SSL certificate file"
)
new_root = None
if server_args.ssl_ca_certs:
new_root = _read_ssl_file(
server_args.ssl_ca_certs,
"SSL CA certificates file",
)
logger.info("gRPC SSL certificate change detected, reloading.")
config = grpc.ssl_server_certificate_configuration(
[(new_key, new_cert)],
root_certificates=new_root,
)
# Update mtimes only after successful reload
_cert_mtime = new_cert_mt
_key_mtime = new_key_mt
_ca_mtime = new_ca_mt
return config
except Exception:
logger.exception(
"Failed to reload gRPC SSL certificates — "
"continuing with previous certificates."
)
return None
try:
initial_config = grpc.ssl_server_certificate_configuration(
[(private_key, certificate_chain)],
root_certificates=root_certificates,
)
credentials = grpc.dynamic_ssl_server_credentials(
initial_config,
_cert_config_fetcher,
)
except Exception as e:
raise ValueError(
f"Failed to create gRPC dynamic SSL credentials. "
f"Verify that --ssl-keyfile and --ssl-certfile contain "
f"valid, matching PEM data. Underlying error: {e}"
) from e
logger.info("gRPC SSL certificate auto-refresh enabled.")
else:
try:
credentials = grpc.ssl_server_credentials(
[(private_key, certificate_chain)], # pairs: (key, cert)
root_certificates=root_certificates,
)
except Exception as e:
raise ValueError(
f"Failed to create gRPC SSL credentials. Verify that "
f"--ssl-keyfile and --ssl-certfile contain valid, matching "
f"PEM data. Underlying error: {e}"
) from e
bound_port = server.add_secure_port(listen_addr, credentials)
if bound_port == 0:
raise RuntimeError(
f"Failed to bind gRPC TLS server to {listen_addr}. "
f"Check that the port is available and SSL credentials are valid."
)
logger.info(f"gRPC server (TLS) listening on {listen_addr}")
else:
server.add_insecure_port(listen_addr)
logger.info(f"gRPC server listening on {listen_addr}")
await server.start()
logger.info(f"gRPC server listening on {listen_addr}")
# Start warmup in a separate thread
warmup_thread = threading.Thread(

84
python/sglang/srt/entrypoints/http_server.py
View File

@@ -1756,12 +1756,16 @@ def _execute_server_warmup(server_args: ServerArgs):
if server_args.api_key:
headers["Authorization"] = f"Bearer {server_args.api_key}"
ssl_verify = server_args.ssl_verify()
# Wait until the server is launched
success = False
for _ in range(120):
time.sleep(1)
try:
res = requests.get(url + "/model_info", timeout=5, headers=headers)
res = requests.get(
url + "/model_info", timeout=5, headers=headers, verify=ssl_verify
)
assert res.status_code == 200, f"{res=}, {res.text=}"
success = True
break
@@ -1850,6 +1854,7 @@ def _execute_server_warmup(server_args: ServerArgs):
json=json_data,
headers=headers,
timeout=warmup_timeout if warmup_timeout > 0 else 600,
verify=ssl_verify,
)
assert res.status_code == 200, f"{res.text}"
_global_state.tokenizer_manager.server_status = ServerStatus.Up
@@ -1878,6 +1883,7 @@ def _execute_server_warmup(server_args: ServerArgs):
timeout=(
warmup_timeout if warmup_timeout > 0 else 1800
), # because of deep gemm precache is very long if not precache.
verify=ssl_verify,
)
if res.status_code == 200:
logger.info(
@@ -2057,17 +2063,64 @@ def launch_server(
# while model/subprocess initialization is still in progress.
reserved_socket.listen(128)
if server_args.ssl_certfile:
logger.info(
f"SSL enabled: certfile={server_args.ssl_certfile}, "
f"keyfile={server_args.ssl_keyfile}"
)
# Listen for HTTP requests
if server_args.tokenizer_worker_num == 1:
# Default case, one tokenizer process
uvicorn.run(
app,
fd=reserved_socket.fileno(),
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=5,
loop="uvloop",
)
if server_args.enable_ssl_refresh:
# Use Config/Server API for access to the SSLContext.
config = uvicorn.Config(
app,
fd=reserved_socket.fileno(),
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=5,
loop="uvloop",
ssl_keyfile=server_args.ssl_keyfile,
ssl_certfile=server_args.ssl_certfile,
ssl_ca_certs=server_args.ssl_ca_certs,
ssl_keyfile_password=server_args.ssl_keyfile_password,
)
config.load() # Creates the SSLContext
from sglang.srt.entrypoints.ssl_utils import SSLCertRefresher
server = uvicorn.Server(config)
async def _run_with_ssl_refresh():
refresher = SSLCertRefresher(
config.ssl,
server_args.ssl_keyfile,
server_args.ssl_certfile,
server_args.ssl_ca_certs,
)
logger.info("SSL certificate auto-refresh enabled.")
try:
await server.serve()
finally:
refresher.stop()
import asyncio
asyncio.run(_run_with_ssl_refresh())
else:
# Default case, one tokenizer process
uvicorn.run(
app,
fd=reserved_socket.fileno(),
root_path=server_args.fastapi_root_path,
log_level=server_args.log_level_http or server_args.log_level,
timeout_keep_alive=5,
loop="uvloop",
ssl_keyfile=server_args.ssl_keyfile,
ssl_certfile=server_args.ssl_certfile,
ssl_ca_certs=server_args.ssl_ca_certs,
ssl_keyfile_password=server_args.ssl_keyfile_password,
)
else:
# Multiple tokenizer and http processes
from uvicorn.config import LOGGING_CONFIG
@@ -2079,6 +2132,13 @@ def launch_server(
}
monkey_patch_uvicorn_multiprocessing()
if server_args.enable_ssl_refresh:
logger.warning(
"--enable-ssl-refresh is not supported with multiple "
"tokenizer workers (--tokenizer-worker-num > 1). "
"SSL refresh will be disabled."
)
uvicorn.run(
"sglang.srt.entrypoints.http_server:app",
fd=reserved_socket.fileno(),
@@ -2087,6 +2147,10 @@ def launch_server(
timeout_keep_alive=5,
loop="uvloop",
workers=server_args.tokenizer_worker_num,
ssl_keyfile=server_args.ssl_keyfile,
ssl_certfile=server_args.ssl_certfile,
ssl_ca_certs=server_args.ssl_ca_certs,
ssl_keyfile_password=server_args.ssl_keyfile_password,
)
finally:
# Close the reserved socket after uvicorn exits or on any error

12
python/sglang/srt/entrypoints/http_server_engine.py
View File

@@ -20,6 +20,8 @@ def launch_server_process(server_args: ServerArgs) -> multiprocessing.Process:
timeout = 300.0 # Increased timeout to 5 minutes for downloading large models
start_time = time.perf_counter()
ssl_verify = server_args.ssl_verify()
with requests.Session() as session:
while time.perf_counter() - start_time < timeout:
try:
@@ -27,7 +29,9 @@ def launch_server_process(server_args: ServerArgs) -> multiprocessing.Process:
"Content-Type": "application/json; charset=utf-8",
"Authorization": f"Bearer {server_args.api_key}",
}
response = session.get(f"{base_url}/health_generate", headers=headers)
response = session.get(
f"{base_url}/health_generate", headers=headers, verify=ssl_verify
)
if response.status_code == 200:
return p
except requests.RequestException:
@@ -64,8 +68,10 @@ class HttpServerEngineAdapter(EngineBase):
Returns:
The JSON response from the server
"""
url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}"
response = requests.post(url, json=payload or {})
url = f"{self.server_args.url()}/{endpoint}"
response = requests.post(
url, json=payload or {}, verify=self.server_args.ssl_verify()
)
response.raise_for_status()
return response.json()

88
python/sglang/srt/entrypoints/ssl_utils.py Normal file
View File

@@ -0,0 +1,88 @@
"""Utilities for SSL certificate hot-reloading."""
import asyncio
import logging
import ssl
from typing import Optional
from watchfiles import awatch
logger = logging.getLogger(__name__)
class SSLCertRefresher:
"""Monitors SSL certificate files and reloads them when changed.
Uses ``watchfiles.awatch()`` for efficient inotify/kqueue-based
file monitoring. On change the referenced :class:`ssl.SSLContext`
is updated in-place so that new TLS connections automatically pick
up the fresh certificates.
"""
def __init__(
self,
ssl_context: ssl.SSLContext,
key_path: str,
cert_path: str,
ca_path: Optional[str] = None,
) -> None:
self._ssl_context = ssl_context
self._key_path = key_path
self._cert_path = cert_path
self._ca_path = ca_path
self._tasks: list[asyncio.Task] = []
loop = asyncio.get_running_loop()
self._tasks.append(
loop.create_task(self._watch_cert_key(), name="ssl-cert-key-watcher")
)
if self._ca_path:
self._tasks.append(
loop.create_task(self._watch_ca(), name="ssl-ca-watcher")
)
async def _watch_cert_key(self) -> None:
"""Watch cert and key files and reload on change."""
try:
async for _changes in awatch(self._cert_path, self._key_path):
logger.info(
"SSL cert/key file change detected, reloading: " "cert=%s key=%s",
self._cert_path,
self._key_path,
)
try:
self._ssl_context.load_cert_chain(self._cert_path, self._key_path)
logger.info("SSL cert/key reloaded successfully.")
except Exception:
logger.exception(
"Failed to reload SSL cert/key — continuing with "
"previous certificates."
)
except asyncio.CancelledError:
return
async def _watch_ca(self) -> None:
"""Watch CA file and reload on change."""
assert self._ca_path is not None
try:
async for _changes in awatch(self._ca_path):
logger.info(
"SSL CA file change detected, reloading: ca=%s",
self._ca_path,
)
try:
self._ssl_context.load_verify_locations(self._ca_path)
logger.info("SSL CA certificates reloaded successfully.")
except Exception:
logger.exception(
"Failed to reload SSL CA certificates — continuing "
"with previous CA bundle."
)
except asyncio.CancelledError:
return
def stop(self) -> None:
"""Cancel all watching tasks."""
for task in self._tasks:
task.cancel()
self._tasks.clear()

123
python/sglang/srt/server_args.py
View File

@@ -309,6 +309,13 @@ class ServerArgs:
nccl_port: Optional[int] = None
checkpoint_engine_wait_weights_before_ready: bool = False
# SSL/TLS
ssl_keyfile: Optional[str] = None
ssl_certfile: Optional[str] = None
ssl_ca_certs: Optional[str] = None
ssl_keyfile_password: Optional[str] = None
enable_ssl_refresh: bool = False
# Quantization and data type
dtype: str = "auto"
quantization: Optional[str] = None
@@ -718,6 +725,9 @@ class ServerArgs:
# Normalize load balancing defaults early (before dummy-model short-circuit).
self._handle_load_balance_method()
# Validate SSL arguments early (before dummy-model short-circuit).
self._handle_ssl_validation()
if self.model_path.lower() in ["none", "dummy"]:
# Skip for dummy models
return
@@ -827,6 +837,47 @@ class ServerArgs:
)
return
def _handle_ssl_validation(self):
"""Ensure SSL arguments are consistent and referenced files exist."""
if self.ssl_keyfile and not self.ssl_certfile:
raise ValueError(
"--ssl-keyfile requires --ssl-certfile to be specified as well."
)
if self.ssl_certfile and not self.ssl_keyfile:
raise ValueError(
"--ssl-certfile requires --ssl-keyfile to be specified as well."
)
if not self.ssl_certfile and not self.ssl_keyfile:
if self.ssl_ca_certs:
raise ValueError(
"--ssl-ca-certs has no effect without --ssl-certfile and --ssl-keyfile."
)
if self.ssl_keyfile_password:
raise ValueError(
"--ssl-keyfile-password has no effect without --ssl-certfile and --ssl-keyfile."
)
# Validate files exist early to avoid late failures after model loading.
if self.ssl_keyfile and not os.path.isfile(self.ssl_keyfile):
raise ValueError(
f"SSL key file not found: '{self.ssl_keyfile}'. "
f"Please check the --ssl-keyfile path."
)
if self.ssl_certfile and not os.path.isfile(self.ssl_certfile):
raise ValueError(
f"SSL certificate file not found: '{self.ssl_certfile}'. "
f"Please check the --ssl-certfile path."
)
if self.ssl_ca_certs and not os.path.isfile(self.ssl_ca_certs):
raise ValueError(
f"SSL CA certificates file not found: '{self.ssl_ca_certs}'. "
f"Please check the --ssl-ca-certs path."
)
if self.enable_ssl_refresh and not (self.ssl_certfile and self.ssl_keyfile):
raise ValueError(
"--enable-ssl-refresh requires --ssl-certfile and --ssl-keyfile "
"to be specified."
)
def _handle_deprecated_args(self):
# Handle deprecated tool call parsers
deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
@@ -3237,6 +3288,39 @@ class ServerArgs:
"before serving inference requests.",
)
# SSL/TLS
parser.add_argument(
"--ssl-keyfile",
type=str,
default=ServerArgs.ssl_keyfile,
help="The file path to the SSL key file.",
)
parser.add_argument(
"--ssl-certfile",
type=str,
default=ServerArgs.ssl_certfile,
help="The file path to the SSL certificate file.",
)
parser.add_argument(
"--ssl-ca-certs",
type=str,
default=ServerArgs.ssl_ca_certs,
help="The CA certificates file.",
)
parser.add_argument(
"--ssl-keyfile-password",
type=str,
default=ServerArgs.ssl_keyfile_password,
help="The password to decrypt the SSL keyfile.",
)
parser.add_argument(
"--enable-ssl-refresh",
action="store_true",
default=ServerArgs.enable_ssl_refresh,
help="Enable automatic SSL certificate hot-reloading when cert/key "
"files change on disk. Requires --ssl-certfile and --ssl-keyfile.",
)
# Quantization and data type
parser.add_argument(
"--dtype",
@@ -5337,10 +5421,43 @@ class ServerArgs:
return cls(**{attr: getattr(args, attr) for attr in attrs})
def url(self):
if is_valid_ipv6_address(self.host):
return f"http://[{self.host}]:{self.port}"
scheme = "https" if self.ssl_certfile else "http"
# When binding to all interfaces, use loopback for internal requests.
host = self.host
if not host or host == "0.0.0.0":
host = "127.0.0.1"
elif host == "::":
host = "::1"
if is_valid_ipv6_address(host):
return f"{scheme}://[{host}]:{self.port}"
else:
return f"http://{self.host}:{self.port}"
return f"{scheme}://{host}:{self.port}"
def ssl_verify(self):
"""Return the value for the requests library's ``verify=`` parameter.
When SSL is configured:
- If a CA certificate file is provided, return its path so requests
validates the server certificate against that CA.
- Otherwise, return False to disable certificate verification
(suitable for self-signed certificates in development/testing).
A warning is logged once when this happens.
When SSL is not configured, return True to use the system's default
CA bundle.
"""
if self.ssl_ca_certs:
return self.ssl_ca_certs
if self.ssl_certfile:
if not getattr(self, "_ssl_verify_warned", False):
logger.warning(
"SSL is enabled but --ssl-ca-certs was not provided. "
"Certificate verification is DISABLED for internal "
"health checks. For production deployments, provide "
"--ssl-ca-certs or use CA-signed certificates."
)
self._ssl_verify_warned = True
return False
return True
def get_model_config(self):
# Lazy init to avoid circular import