diff --git a/sgl-model-gateway/src/core/job_queue.rs b/sgl-model-gateway/src/core/job_queue.rs index a98486ae9..bb928a836 100644 --- a/sgl-model-gateway/src/core/job_queue.rs +++ b/sgl-model-gateway/src/core/job_queue.rs @@ -578,7 +578,7 @@ impl JobQueue { tokenizer_path: None, reasoning_parser: None, tool_parser: None, - chat_template: None, + chat_template: router_config.chat_template.clone(), bootstrap_port: None, health_check_timeout_secs: router_config.health_check.timeout_secs, health_check_interval_secs: router_config @@ -649,7 +649,7 @@ impl JobQueue { tokenizer_path: None, reasoning_parser: None, tool_parser: None, - chat_template: None, + chat_template: router_config.chat_template.clone(), bootstrap_port, health_check_timeout_secs: router_config.health_check.timeout_secs, health_check_interval_secs: router_config.health_check.check_interval_secs, diff --git a/sgl-model-gateway/src/core/steps/worker/local/register_tokenizer.rs b/sgl-model-gateway/src/core/steps/worker/local/register_tokenizer.rs index a6c401e02..a1fb5923a 100644 --- a/sgl-model-gateway/src/core/steps/worker/local/register_tokenizer.rs +++ b/sgl-model-gateway/src/core/steps/worker/local/register_tokenizer.rs @@ -29,6 +29,13 @@ impl StepExecutor for RegisterTokenizerStep { .actual_workers .as_ref() .ok_or_else(|| WorkflowError::ContextValueNotFound("workers".to_string()))?; + // Get chat_template: worker config > global router config + let chat_template = context + .data + .config + .chat_template + .clone() + .or_else(|| app_context.router_config.chat_template.clone()); for worker in workers.iter() { let model_id = worker.model_id().to_string(); @@ -55,12 +62,17 @@ impl StepExecutor for RegisterTokenizerStep { // Load tokenizer with thread safe lock let tokenizer_path_owned = tokenizer_path.clone(); + let template = chat_template.clone(); if let Err(e) = app_context .tokenizer_registry - .load(&tokenizer_id, &model_id, &source, || async move { - factory::create_tokenizer_async(&tokenizer_path_owned) - .await - .map_err(|e| e.to_string()) + .load(&tokenizer_id, &model_id, &source, move || { + let path = tokenizer_path_owned; + let tmpl = template; + async move { + factory::create_tokenizer_async_with_chat_template(&path, tmpl.as_deref()) + .await + .map_err(|e| e.to_string()) + } }) .await {