[router] add tokenizer chat template support (#9370)

Co-authored-by: Chang Su <chang.s.su@oracle.com>
This commit is contained in:
Simo Lin
2025-08-19 20:14:02 -07:00
committed by GitHub
co-authored by Chang Su
parent 7638f5e44e
commit 5fbad308cd
12 changed files with 748 additions and 85 deletions
+5 -5
View File
@@ -1,4 +1,4 @@
use super::traits::Tokenizer as TokenizerTrait;
use super::traits::{TokenIdType, Tokenizer as TokenizerTrait};
use anyhow::Result;
use std::sync::Arc;
@@ -9,7 +9,7 @@ pub struct Sequence {
tokenizer: Arc<dyn TokenizerTrait>,
/// The current sequence of token ids
token_ids: Vec<u32>,
token_ids: Vec<TokenIdType>,
/// The position in the current sequence the last decoded token completed
prefix_offset: usize,
@@ -54,7 +54,7 @@ impl Sequence {
}
/// Create a sequence with initial tokens
pub fn with_tokens(tokenizer: Arc<dyn TokenizerTrait>, token_ids: Vec<u32>) -> Self {
pub fn with_tokens(tokenizer: Arc<dyn TokenizerTrait>, token_ids: Vec<TokenIdType>) -> Self {
let len = token_ids.len();
Self {
tokenizer,
@@ -90,7 +90,7 @@ impl Sequence {
/// Append a single token to the sequence and return newly decoded text
/// Based on HuggingFace TGI incremental decoding
pub fn append_token(&mut self, token_id: u32) -> Result<String> {
pub fn append_token(&mut self, token_id: TokenIdType) -> Result<String> {
// Store the old read offset before adding the new token
let old_read_offset = self.read_offset;
@@ -145,7 +145,7 @@ impl Sequence {
}
/// Get the current token ids
pub fn token_ids(&self) -> &[u32] {
pub fn token_ids(&self) -> &[TokenIdType] {
&self.token_ids
}