chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Model-Gateway: grpc]: create tokenizer with chat template (#17052)

Browse Source 
Signed-off-by: jeff.ye <jeff.ye@novita.ai>
This commit is contained in:
jeff.ye
2026-01-14 17:56:08 -08:00
committed by GitHub
parent 4346db5faf
commit c020d30045
2 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-model-gateway/src/core/job_queue.rs
+2 -2
View File
@@ -578,7 +578,7 @@ impl JobQueue {
tokenizer_path: None,
reasoning_parser: None,
tool_parser: None,
chat_template: None,
chat_template: router_config.chat_template.clone(),
bootstrap_port: None,
health_check_timeout_secs: router_config.health_check.timeout_secs,
health_check_interval_secs: router_config
@@ -649,7 +649,7 @@ impl JobQueue {
tokenizer_path: None,
reasoning_parser: None,
tool_parser: None,
chat_template: None,
chat_template: router_config.chat_template.clone(),
bootstrap_port,
health_check_timeout_secs: router_config.health_check.timeout_secs,
health_check_interval_secs: router_config.health_check.check_interval_secs,
sgl-model-gateway/src/core/steps/worker/local/register_tokenizer.rs
+16 -4
View File
@@ -29,6 +29,13 @@ impl StepExecutor<LocalWorkerWorkflowData> for RegisterTokenizerStep {
.actual_workers
.as_ref()
.ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?;
// Get chat_template: worker config > global router config
let chat_template = context
.data
.config
.chat_template
.clone()
.or_else(|| app_context.router_config.chat_template.clone());
for worker in workers.iter() {
let model_id = worker.model_id().to_string();
@@ -55,12 +62,17 @@ impl StepExecutor<LocalWorkerWorkflowData> for RegisterTokenizerStep {
// Load tokenizer with thread safe lock
let tokenizer_path_owned = tokenizer_path.clone();
let template = chat_template.clone();
if let Err(e) = app_context
.tokenizer_registry
.load(&tokenizer_id, &model_id, &source, || async move {
factory::create_tokenizer_async(&tokenizer_path_owned)
.await
.map_err(|e| e.to_string())
.load(&tokenizer_id, &model_id, &source, move || {
let path = tokenizer_path_owned;
let tmpl = template;
async move {
factory::create_tokenizer_async_with_chat_template(&path, tmpl.as_deref())
.await
.map_err(|e| e.to_string())
}
})
.await
{