[smg] use official tokenizer crate instead of manually built one (#17721)

2026-01-25 13:35:16 -08:00
parent 2c1e2674cc
commit f5ac1ca10b
4 changed files with 2 additions and 2794 deletions
@@ -15,7 +15,7 @@ pub use reasoning_parser;
 pub mod routers;
 pub mod server;
 pub mod service_discovery;
-pub mod tokenizer;
+pub use llm_tokenizer as tokenizer;
 pub use tool_parser;
 pub mod version;
 pub mod wasm;