diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e65be3f36..548ca5d15 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -25,23 +25,23 @@ /python/sglang/srt/multimodal @mickqian @JustinTong0323 @yhyang201 @yuan-luo /python/sglang/srt/speculative @Ying1123 @merrymercy @hnyls2002 /sgl-kernel @zhyncs @ispobock @BBuf @yizhang2077 @merrymercy @FlamingoPg @HaiShaw -/sgl-router @slin1237 @CatherineSue -/sgl-router/benches @slin1237 -/sgl-router/bindings/python @CatherineSue @key4ng @slin1237 -/sgl-router/py_test @CatherineSue @key4ng -/sgl-router/src/config @slin1237 -/sgl-router/src/core @slin1237 -/sgl-router/src/data_connector @key4ng -/sgl-router/src/grpc_client @CatherineSue @slin1237 -/sgl-router/src/mcp @key4ng @slin1237 -/sgl-router/src/policies @slin1237 @ByronHsu -/sgl-router/src/proto @CatherineSue @slin1237 -/sgl-router/src/protocols @CatherineSue @key4ng -/sgl-router/src/reasoning_parser @CatherineSue -/sgl-router/src/routers @CatherineSue @key4ng @slin1237 -/sgl-router/src/tokenizer @slin1237 @CatherineSue -/sgl-router/src/tool_parser @slin1237 @CatherineSue -/sgl-router/src/wasm @tonyluj -/sgl-router/examples/wasm @tonyluj +/sgl-model-gateway @slin1237 @CatherineSue +/sgl-model-gateway/benches @slin1237 +/sgl-model-gateway/bindings/python @CatherineSue @key4ng @slin1237 +/sgl-model-gateway/py_test @CatherineSue @key4ng +/sgl-model-gateway/src/config @slin1237 +/sgl-model-gateway/src/core @slin1237 +/sgl-model-gateway/src/data_connector @key4ng +/sgl-model-gateway/src/grpc_client @CatherineSue @slin1237 +/sgl-model-gateway/src/mcp @key4ng @slin1237 +/sgl-model-gateway/src/policies @slin1237 @ByronHsu +/sgl-model-gateway/src/proto @CatherineSue @slin1237 +/sgl-model-gateway/src/protocols @CatherineSue @key4ng +/sgl-model-gateway/src/reasoning_parser @CatherineSue +/sgl-model-gateway/src/routers @CatherineSue @key4ng @slin1237 +/sgl-model-gateway/src/tokenizer @slin1237 @CatherineSue +/sgl-model-gateway/src/tool_parser @slin1237 @CatherineSue +/sgl-model-gateway/src/wasm @tonyluj +/sgl-model-gateway/examples/wasm @tonyluj /test/srt/ascend @ping1jing2 @iforgetmyname /test/srt/test_modelopt* @Edwardf0t1 diff --git a/.gitignore b/.gitignore index 93947f3dd..532af84b6 100644 --- a/.gitignore +++ b/.gitignore @@ -241,7 +241,7 @@ compile_commands.json Cargo.lock # Generated vision test fixtures (regenerate with: python scripts/generate_vision_golden.py) -sgl-router/tests/fixtures/golden/ +sgl-model-gateway/tests/fixtures/golden/ lmms-eval diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index b7fcf770e..1fa9e7216 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -199,14 +199,14 @@ RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ -type f -name '*MI300X*' | xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} -# Install Rust toolchain for sgl-router +# Install Rust toolchain for sgl-model-gateway ENV PATH="/root/.cargo/bin:${PATH}" RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ && rustc --version && cargo --version -# Build and install sgl-router +# Build and install sgl-model-gateway RUN python3 -m pip install --no-cache-dir setuptools-rust \ - && cd /sgl-workspace/sglang/sgl-router/bindings/python \ + && cd /sgl-workspace/sglang/sgl-model-gateway/bindings/python \ && cargo build --release \ && python3 -m pip install --no-cache-dir . \ && rm -rf /root/.cache diff --git a/sgl-model-gateway/examples/wasm/README.md b/sgl-model-gateway/examples/wasm/README.md index b82dae78f..470e64191 100644 --- a/sgl-model-gateway/examples/wasm/README.md +++ b/sgl-model-gateway/examples/wasm/README.md @@ -1,6 +1,6 @@ -# WASM Guest Examples for sgl-router +# WASM Guest Examples for sgl-model-gateway -This directory contains example WASM middleware components demonstrating how to implement custom middleware for sgl-router using the WebAssembly Component Model. +This directory contains example WASM middleware components demonstrating how to implement custom middleware for sgl-model-gateway using the WebAssembly Component Model. ## Examples Overview @@ -56,7 +56,7 @@ All examples require: - Rust toolchain (latest stable) - `wasm32-wasip2` target: `rustup target add wasm32-wasip2` - `wasm-tools`: `cargo install wasm-tools` -- sgl-router running with WASM enabled (`--enable-wasm`) +- sgl-model-gateway running with WASM enabled (`--enable-wasm`) ## Building All Examples diff --git a/sgl-model-gateway/examples/wasm/wasm-guest-auth/README.md b/sgl-model-gateway/examples/wasm/wasm-guest-auth/README.md index 9e2723f24..a931243db 100644 --- a/sgl-model-gateway/examples/wasm/wasm-guest-auth/README.md +++ b/sgl-model-gateway/examples/wasm/wasm-guest-auth/README.md @@ -1,6 +1,6 @@ -# WASM Auth Example for sgl-router +# WASM Auth Example for sgl-model-gateway -This example demonstrates API key authentication middleware for sgl-router using the WebAssembly Component Model. +This example demonstrates API key authentication middleware for sgl-model-gateway using the WebAssembly Component Model. ## Overview diff --git a/sgl-model-gateway/examples/wasm/wasm-guest-logging/README.md b/sgl-model-gateway/examples/wasm/wasm-guest-logging/README.md index 4e216a98b..35316fc88 100644 --- a/sgl-model-gateway/examples/wasm/wasm-guest-logging/README.md +++ b/sgl-model-gateway/examples/wasm/wasm-guest-logging/README.md @@ -1,6 +1,6 @@ -# WASM Logging Example for sgl-router +# WASM Logging Example for sgl-model-gateway -This example demonstrates logging and tracing middleware for sgl-router using the WebAssembly Component Model. +This example demonstrates logging and tracing middleware for sgl-model-gateway using the WebAssembly Component Model. ## Overview diff --git a/sgl-model-gateway/examples/wasm/wasm-guest-ratelimit/README.md b/sgl-model-gateway/examples/wasm/wasm-guest-ratelimit/README.md index d85a30ab5..e7577b7a0 100644 --- a/sgl-model-gateway/examples/wasm/wasm-guest-ratelimit/README.md +++ b/sgl-model-gateway/examples/wasm/wasm-guest-ratelimit/README.md @@ -1,6 +1,6 @@ -# WASM Rate Limit Example for sgl-router +# WASM Rate Limit Example for sgl-model-gateway -This example demonstrates rate limiting middleware for sgl-router using the WebAssembly Component Model. +This example demonstrates rate limiting middleware for sgl-model-gateway using the WebAssembly Component Model. ## Overview diff --git a/sgl-model-gateway/src/middleware.rs b/sgl-model-gateway/src/middleware.rs index d35ae2fe5..bf8884822 100644 --- a/sgl-model-gateway/src/middleware.rs +++ b/sgl-model-gateway/src/middleware.rs @@ -28,7 +28,7 @@ use crate::{ module::{MiddlewareAttachPoint, WasmModuleAttachPoint}, spec::{ apply_modify_action_to_headers, build_wasm_headers_from_axum_headers, - sgl::router::middleware_types::{ + sgl::model_gateway::middleware_types::{ Action, Request as WasmRequest, Response as WasmResponse, }, }, diff --git a/sgl-model-gateway/src/wasm/README.md b/sgl-model-gateway/src/wasm/README.md index caf7837ef..7bd898fe4 100644 --- a/sgl-model-gateway/src/wasm/README.md +++ b/sgl-model-gateway/src/wasm/README.md @@ -1,10 +1,10 @@ -# WebAssembly (WASM) Extensibility for sgl-router +# WebAssembly (WASM) Extensibility for sgl-model-gateway -This module provides WebAssembly-based extensibility for sgl-router, enabling dynamic, safe, and portable middleware execution without requiring router restarts or recompilation. +This module provides WebAssembly-based extensibility for sgl-model-gateway, enabling dynamic, safe, and portable middleware execution without requiring router restarts or recompilation. ## Overview -The WASM module allows you to extend sgl-router functionality by deploying WebAssembly components that can: +The WASM module allows you to extend sgl-model-gateway functionality by deploying WebAssembly components that can: - **Intercept requests/responses** at various lifecycle points (OnRequest, OnResponse) - **Modify HTTP headers and bodies** before/after processing @@ -68,7 +68,7 @@ See [`interface/`](./interface/) for the complete interface definition. ### Prerequisites -- sgl-router compiled with WASM support +- sgl-model-gateway compiled with WASM support - Rust toolchain (for building WASM components) - `wasm32-wasip2` target: `rustup target add wasm32-wasip2` - `wasm-tools`: `cargo install wasm-tools` @@ -78,7 +78,7 @@ See [`interface/`](./interface/) for the complete interface definition. Enable WASM support when starting the router: ```bash -./sgl-router --enable-wasm --worker-urls=http://0.0.0.0:30000 --port=3000 +./sgl-model-gateway --enable-wasm --worker-urls=http://0.0.0.0:30000 --port=3000 ``` ### Deploying a WASM Module @@ -208,11 +208,11 @@ Define your component using the WASM interface from `interface/spec.*`: ```rust wit_bindgen::generate!({ path: "../../../src/wasm/interface", - world: "sgl-router", + world: "sgl-model-gateway", }); -use exports::sgl::router::middleware_on_request::Guest as OnRequestGuest; -use sgl::router::middleware_types::{Request, Action}; +use exports::sgl::model_gateway::middleware_on_request::Guest as OnRequestGuest; +use sgl::model_gateway::middleware_types::{Request, Action}; struct Middleware; diff --git a/sgl-model-gateway/src/wasm/interface/spec.wit b/sgl-model-gateway/src/wasm/interface/spec.wit index 1f865c971..7750a32da 100644 --- a/sgl-model-gateway/src/wasm/interface/spec.wit +++ b/sgl-model-gateway/src/wasm/interface/spec.wit @@ -1,4 +1,4 @@ -package sgl:router; +package sgl:model-gateway; interface middleware-types { record header { name: string, value: string } @@ -48,7 +48,7 @@ interface middleware-on-response { on-response: func(resp: response) -> action; } -world sgl-router { +world sgl-model-gateway { export middleware-on-request; export middleware-on-response; } diff --git a/sgl-model-gateway/src/wasm/mod.rs b/sgl-model-gateway/src/wasm/mod.rs index fe88d55f5..fa8d55aac 100644 --- a/sgl-model-gateway/src/wasm/mod.rs +++ b/sgl-model-gateway/src/wasm/mod.rs @@ -1,4 +1,4 @@ -//! WebAssembly (WASM) module support for sgl-router +//! WebAssembly (WASM) module support for sgl-model-gateway //! //! This module provides WASM component execution capabilities using the WebAssembly Component Model. //! It supports middleware execution at various attach points (OnRequest, OnResponse) with async support. diff --git a/sgl-model-gateway/src/wasm/module_manager.rs b/sgl-model-gateway/src/wasm/module_manager.rs index 23aee12d9..7fc9876e1 100644 --- a/sgl-model-gateway/src/wasm/module_manager.rs +++ b/sgl-model-gateway/src/wasm/module_manager.rs @@ -227,7 +227,7 @@ impl WasmModuleManager { module: &WasmModule, attach_point: WasmModuleAttachPoint, input: WasmComponentInput, - ) -> Option { + ) -> Option { use tracing::error; let action_result = self diff --git a/sgl-model-gateway/src/wasm/runtime.rs b/sgl-model-gateway/src/wasm/runtime.rs index 7b610d040..2223b0968 100644 --- a/sgl-model-gateway/src/wasm/runtime.rs +++ b/sgl-model-gateway/src/wasm/runtime.rs @@ -20,7 +20,7 @@ use crate::wasm::{ config::WasmRuntimeConfig, errors::{Result, WasmError, WasmRuntimeError}, module::{MiddlewareAttachPoint, WasmModuleAttachPoint}, - spec::SglRouter, + spec::SglModelGateway, types::{WasiState, WasmComponentInput, WasmComponentOutput}, }; @@ -322,7 +322,7 @@ impl WasmThreadPool { }; // Instantiate component (must use async instantiation when async support is enabled) - let bindings = SglRouter::instantiate_async(&mut store, &component, &linker) + let bindings = SglModelGateway::instantiate_async(&mut store, &component, &linker) .await .map_err(|e| { WasmError::from(WasmRuntimeError::InstanceCreateFailed(e.to_string())) @@ -330,7 +330,7 @@ impl WasmThreadPool { // Call on-request (async call when async support is enabled) let action_result = bindings - .sgl_router_middleware_on_request() + .sgl_model_gateway_middleware_on_request() .call_on_request(&mut store, &request) .await .map_err(|e| WasmError::from(WasmRuntimeError::CallFailed(e.to_string())))?; @@ -350,7 +350,7 @@ impl WasmThreadPool { }; // Instantiate component (must use async instantiation when async support is enabled) - let bindings = SglRouter::instantiate_async(&mut store, &component, &linker) + let bindings = SglModelGateway::instantiate_async(&mut store, &component, &linker) .await .map_err(|e| { WasmError::from(WasmRuntimeError::InstanceCreateFailed(e.to_string())) @@ -358,7 +358,7 @@ impl WasmThreadPool { // Call on-response (async call when async support is enabled) let action_result = bindings - .sgl_router_middleware_on_response() + .sgl_model_gateway_middleware_on_response() .call_on_response(&mut store, &response) .await .map_err(|e| WasmError::from(WasmRuntimeError::CallFailed(e.to_string())))?; diff --git a/sgl-model-gateway/src/wasm/spec.rs b/sgl-model-gateway/src/wasm/spec.rs index b43c78ddd..263002b1f 100644 --- a/sgl-model-gateway/src/wasm/spec.rs +++ b/sgl-model-gateway/src/wasm/spec.rs @@ -7,7 +7,7 @@ use axum::http::{header, HeaderMap, HeaderValue}; wasmtime::component::bindgen!({ path: "src/wasm/interface", - world: "sgl-router", + world: "sgl-model-gateway", imports: { default: async | trappable }, exports: { default: async }, }); @@ -15,11 +15,11 @@ wasmtime::component::bindgen!({ /// Build WebAssembly headers from Axum HeaderMap pub fn build_wasm_headers_from_axum_headers( headers: &HeaderMap, -) -> Vec { +) -> Vec { let mut wasm_headers = Vec::new(); for (name, value) in headers.iter() { if let Ok(value_str) = value.to_str() { - wasm_headers.push(sgl::router::middleware_types::Header { + wasm_headers.push(sgl::model_gateway::middleware_types::Header { name: name.as_str().to_string(), value: value_str.to_string(), }); @@ -31,7 +31,7 @@ pub fn build_wasm_headers_from_axum_headers( /// Apply ModifyAction header modifications to Axum HeaderMap pub fn apply_modify_action_to_headers( headers: &mut HeaderMap, - modify: &sgl::router::middleware_types::ModifyAction, + modify: &sgl::model_gateway::middleware_types::ModifyAction, ) { // Apply headers_set for header_mod in &modify.headers_set { diff --git a/sgl-model-gateway/src/wasm/types.rs b/sgl-model-gateway/src/wasm/types.rs index 5eaca82bf..261280d5b 100644 --- a/sgl-model-gateway/src/wasm/types.rs +++ b/sgl-model-gateway/src/wasm/types.rs @@ -8,7 +8,7 @@ use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; use crate::wasm::{ module::{MiddlewareAttachPoint, WasmModuleAttachPoint}, - spec::sgl::router::middleware_types, + spec::sgl::model_gateway::middleware_types, }; /// Generic input type for WASM component execution diff --git a/sgl-model-gateway/tests/wasm_test.rs b/sgl-model-gateway/tests/wasm_test.rs index 9beaa19cb..bdd4e0060 100644 --- a/sgl-model-gateway/tests/wasm_test.rs +++ b/sgl-model-gateway/tests/wasm_test.rs @@ -756,7 +756,7 @@ async fn test_wasm_module_execution() { // Execute the module use sgl_model_gateway::wasm::{ - spec::sgl::router::middleware_types, + spec::sgl::model_gateway::middleware_types, types::{WasmComponentInput, WasmComponentOutput}, };