[router] Refactor StopSequenceDecoder to Use Sequence for Incremental Decoding (#11676)

sgl-router/src/tokenizer/sequence.rs

@@ -16,6 +16,9 @@ pub struct Sequence {
 
     /// Current position in the sequence
     read_offset: usize,
+
+    /// Whether to skip special tokens when decoding
+    skip_special_tokens: bool,
 }
 
 impl std::fmt::Debug for Sequence {
@@ -45,22 +48,38 @@ impl std::fmt::Debug for Sequence {
 impl Sequence {
     /// Create a new empty sequence
     pub fn new(tokenizer: Arc<dyn TokenizerTrait>) -> Self {
+        Self::new_with_options(tokenizer, false)
+    }
+
+    /// Create a new empty sequence with skip_special_tokens option
+    pub fn new_with_options(tokenizer: Arc<dyn TokenizerTrait>, skip_special_tokens: bool) -> Self {
         Self {
             tokenizer,
             token_ids: Vec::new(),
             prefix_offset: 0,
             read_offset: 0,
+            skip_special_tokens,
         }
     }
 
     /// Create a sequence with initial tokens
     pub fn with_tokens(tokenizer: Arc<dyn TokenizerTrait>, token_ids: Vec<TokenIdType>) -> Self {
+        Self::with_tokens_and_options(tokenizer, token_ids, false)
+    }
+
+    /// Create a sequence with initial tokens and skip_special_tokens option
+    pub fn with_tokens_and_options(
+        tokenizer: Arc<dyn TokenizerTrait>,
+        token_ids: Vec<TokenIdType>,
+        skip_special_tokens: bool,
+    ) -> Self {
         let len = token_ids.len();
         Self {
             tokenizer,
             token_ids,
             prefix_offset: 0,
             read_offset: len,
+            skip_special_tokens,
         }
     }
 
@@ -99,7 +118,9 @@ impl Sequence {
 
         // If this is the first token or we're at the beginning, decode everything
         if self.prefix_offset == 0 && old_read_offset == 0 {
-            let text = self.tokenizer.decode(&self.token_ids, false)?;
+            let text = self
+                .tokenizer
+                .decode(&self.token_ids, self.skip_special_tokens)?;
             if text.ends_with("�") {
                 // Incomplete UTF-8 sequence, wait for more tokens
                 return Ok(String::new());
@@ -109,14 +130,16 @@ impl Sequence {
         }
 
         // Decode the text up to the previous position
-        let prefix_text = self
-            .tokenizer
-            .decode(&self.token_ids[self.prefix_offset..old_read_offset], false)?;
+        let prefix_text = self.tokenizer.decode(
+            &self.token_ids[self.prefix_offset..old_read_offset],
+            self.skip_special_tokens,
+        )?;
 
         // Decode the text including the new token
-        let new_text = self
-            .tokenizer
-            .decode(&self.token_ids[self.prefix_offset..], false)?;
+        let new_text = self.tokenizer.decode(
+            &self.token_ids[self.prefix_offset..],
+            self.skip_special_tokens,
+        )?;
 
         // Handle multi-byte character boundaries
         let mut prefix_text_len = prefix_text.len();
@@ -151,7 +174,8 @@ impl Sequence {
 
     /// Decode the entire sequence to text
     pub fn text(&self) -> Result<String> {
-        self.tokenizer.decode(&self.token_ids, false)
+        self.tokenizer
+            .decode(&self.token_ids, self.skip_special_tokens)
     }
 
     /// Get the prefix offset
@@ -163,6 +187,11 @@ impl Sequence {
     pub fn read_offset(&self) -> usize {
         self.read_offset
     }
+
+    /// Get whether special tokens are skipped during decoding
+    pub fn skip_special_tokens(&self) -> bool {
+        self.skip_special_tokens
+    }
 }
 
 #[cfg(test)]