[model-gateway] Refine TokenizerRegisty.load() to handle duplication (#17230)

This commit is contained in:
Chang Su
2026-01-16 13:53:21 -08:00
committed by GitHub
parent 2e14407983
commit 4229de3b13
8 changed files with 343 additions and 128 deletions
+1 -1
View File
@@ -27,7 +27,7 @@ mod tests;
use factory::{create_tokenizer_from_file, create_tokenizer_with_chat_template};
// Re-export types used outside this module
pub use huggingface::HuggingFaceTokenizer;
pub use registry::TokenizerRegistry;
pub use registry::{LoadError, LoadOutcome, TokenizerRegistry};
pub use stop::StopSequenceDecoder;
pub use stream::DecodeStream;
pub use traits::{Decoder, Encoder, Encoding, SpecialTokens};
+236 -21
View File
@@ -19,12 +19,51 @@
use std::sync::Arc;
use dashmap::DashMap;
use thiserror::Error;
use tokio::sync::Mutex;
use tracing::{debug, info};
use uuid::Uuid;
use super::traits::Tokenizer;
/// Outcome of a tokenizer load operation
#[derive(Debug, Clone)]
pub enum LoadOutcome {
/// Tokenizer was newly loaded and registered
Loaded { id: String },
/// Tokenizer already existed, returning existing ID
AlreadyExists { id: String },
}
impl LoadOutcome {
/// Get the ID regardless of outcome
pub fn id(&self) -> &str {
match self {
LoadOutcome::Loaded { id } => id,
LoadOutcome::AlreadyExists { id } => id,
}
}
/// Returns true if the tokenizer was newly loaded
pub fn is_newly_loaded(&self) -> bool {
matches!(self, LoadOutcome::Loaded { .. })
}
}
/// Error type for tokenizer loading operations
#[derive(Debug, Error)]
pub enum LoadError {
/// Name cannot be empty
#[error("tokenizer name cannot be empty")]
EmptyName,
/// Source cannot be empty
#[error("tokenizer source cannot be empty")]
EmptySource,
/// Loading failed
#[error("{0}")]
LoadFailed(String),
}
/// Metadata and tokenizer instance for a registered tokenizer
#[derive(Clone)]
pub struct TokenizerEntry {
@@ -53,6 +92,19 @@ pub struct TokenizerRegistry {
loading_locks: DashMap<String, Arc<Mutex<()>>>,
}
/// RAII guard that removes the loading lock entry on drop.
/// Ensures cleanup happens on normal completion, early return, or panic.
struct LoadingLockGuard<'a> {
locks: &'a DashMap<String, Arc<Mutex<()>>>,
key: String,
}
impl Drop for LoadingLockGuard<'_> {
fn drop(&mut self) {
self.locks.remove(&self.key);
}
}
impl TokenizerRegistry {
/// Create a new empty registry
pub fn new() -> Self {
@@ -68,36 +120,46 @@ impl TokenizerRegistry {
Uuid::new_v4().to_string()
}
/// Load and register a tokenizer with a pre-generated ID
/// Load and register a tokenizer
///
/// If the tokenizer is already loaded (by name), returns the existing ID.
/// Otherwise, uses the provided loader function to load it.
/// Validates inputs, handles deduplication, and loads the tokenizer if needed.
/// Per-key locking ensures only one load happens per name, preventing race conditions.
///
/// # Arguments
/// * `id` - Pre-generated UUID for this tokenizer
/// * `name` - User-provided name
/// * `source` - Source path or HuggingFace model ID
/// * `id` - Pre-generated UUID (use `generate_id()` to create one)
/// * `name` - User-provided name (used for deduplication, must not be empty)
/// * `source` - Source path or HuggingFace model ID (must not be empty)
/// * `loader` - Async function that loads the tokenizer
///
/// # Returns
/// * `Ok(id)` - Successfully loaded and registered (returns the ID)
/// * `Err(message)` - Error message if loading fails
/// * `Ok(LoadOutcome::Loaded { id })` - Tokenizer was newly loaded
/// * `Ok(LoadOutcome::AlreadyExists { id })` - Tokenizer already existed
/// * `Err(LoadError)` - Validation failed or loading failed
pub async fn load<F, Fut>(
&self,
id: &str,
name: &str,
source: &str,
loader: F,
) -> Result<String, String>
) -> Result<LoadOutcome, LoadError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<Arc<dyn Tokenizer>, String>>,
{
// Validate inputs
if name.is_empty() {
return Err(LoadError::EmptyName);
}
if source.is_empty() {
return Err(LoadError::EmptySource);
}
// Fast path: already loaded by name
if let Some(existing_id) = self.name_to_id.get(name) {
debug!("Tokenizer already registered for name: {}", name);
return Ok(existing_id.clone());
return Ok(LoadOutcome::AlreadyExists {
id: existing_id.clone(),
});
}
debug!("Tokenizer cache miss for name: {}", name);
@@ -109,24 +171,27 @@ impl TokenizerRegistry {
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone();
let _guard = lock.lock().await;
let _mutex_guard = lock.lock().await;
let _lock_cleanup = LoadingLockGuard {
locks: &self.loading_locks,
key: name.to_string(),
};
// Double-check after acquiring lock (another thread may have loaded it)
if let Some(existing_id) = self.name_to_id.get(name) {
debug!("Tokenizer loaded by another thread for name: {}", name);
return Ok(existing_id.clone());
return Ok(LoadOutcome::AlreadyExists {
id: existing_id.clone(),
});
}
// Load tokenizer
info!("Loading tokenizer '{}' from source: {}", name, source);
let result = loader().await;
// Always clean up the lock, whether loading succeeded or failed
self.loading_locks.remove(name);
let tokenizer = result.map_err(LoadError::LoadFailed)?;
let tokenizer = result?;
// Create entry
// Create entry with provided ID
let entry = TokenizerEntry {
id: id.to_string(),
name: name.to_string(),
@@ -143,18 +208,21 @@ impl TokenizerRegistry {
name, id
);
Ok(id.to_string())
Ok(LoadOutcome::Loaded { id: id.to_string() })
}
/// Register a pre-loaded tokenizer with a pre-generated ID
/// Register a preloaded tokenizer with a pre-generated ID
///
/// Atomically inserts a tokenizer into the registry only if no tokenizer
/// with the same name exists. Returns the ID if successful.
///
/// This is primarily used for testing. Production code should use `load()`.
///
/// # Returns
/// * `Some(id)` - If the tokenizer was successfully registered
/// * `None` - If a tokenizer with this name already existed
pub fn register(
#[cfg(test)]
pub(crate) fn register(
&self,
id: &str,
name: &str,
@@ -292,13 +360,17 @@ mod tests {
// Load and register a tokenizer
let id = TokenizerRegistry::generate_id();
registry
let outcome = registry
.load(&id, "model1", "path/to/model", || async {
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
})
.await
.unwrap();
// Verify LoadOutcome::Loaded
assert!(outcome.is_newly_loaded());
assert_eq!(outcome.id(), id);
// Verify it's loaded
assert!(!registry.is_empty());
assert_eq!(registry.len(), 1);
@@ -317,6 +389,65 @@ mod tests {
assert!(registry.is_empty());
}
#[tokio::test]
async fn test_load_returns_already_exists() {
let registry = TokenizerRegistry::new();
let id1 = TokenizerRegistry::generate_id();
let id2 = TokenizerRegistry::generate_id();
// First load should return Loaded
let outcome1 = registry
.load(&id1, "model1", "source1", || async {
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
})
.await
.unwrap();
assert!(outcome1.is_newly_loaded());
assert_eq!(outcome1.id(), id1);
// Second load with same name should return AlreadyExists with ORIGINAL id
let outcome2 = registry
.load(&id2, "model1", "source2", || async {
panic!("Loader should not be called for duplicate name");
})
.await
.unwrap();
assert!(!outcome2.is_newly_loaded());
assert_eq!(outcome2.id(), id1); // Returns the original ID, not id2
// Registry still has only one entry
assert_eq!(registry.len(), 1);
// Original source is preserved
let entry = registry.get_by_name("model1").unwrap();
assert_eq!(entry.source, "source1");
}
#[tokio::test]
async fn test_load_validation() {
let registry = TokenizerRegistry::new();
let id = TokenizerRegistry::generate_id();
// Empty name should fail
let result = registry
.load(&id, "", "source", || async {
panic!("Loader should not be called for invalid input");
})
.await;
assert!(matches!(result, Err(LoadError::EmptyName)));
// Empty source should fail
let result = registry
.load(&id, "model", "", || async {
panic!("Loader should not be called for invalid input");
})
.await;
assert!(matches!(result, Err(LoadError::EmptySource)));
// Registry should be empty (nothing was loaded)
assert!(registry.is_empty());
}
#[tokio::test]
async fn test_load_prevents_duplicate_loading() {
let registry = Arc::new(TokenizerRegistry::new());
@@ -459,4 +590,88 @@ mod tests {
assert!(result3.is_some());
assert_eq!(registry.len(), 2);
}
#[tokio::test]
async fn test_loading_lock_cleanup_on_panic() {
let registry = Arc::new(TokenizerRegistry::new());
// Spawn a task that will panic during loading
let registry_clone = registry.clone();
let handle = tokio::spawn(async move {
registry_clone
.load(
&TokenizerRegistry::generate_id(),
"panic-model",
"source",
|| async {
panic!("Simulated panic during tokenizer loading");
},
)
.await
});
// Wait for the task - it should panic
let result = handle.await;
assert!(result.is_err(), "Task should have panicked");
// The RAII guard should have cleaned up the loading lock.
// Verify by attempting another load with the same name - it should work,
// not deadlock or fail due to stale lock.
let id = TokenizerRegistry::generate_id();
let outcome = registry
.load(&id, "panic-model", "source", || async {
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
})
.await;
// Should succeed - the lock was properly cleaned up
assert!(outcome.is_ok(), "Load should succeed after panic cleanup");
assert!(outcome.unwrap().is_newly_loaded());
assert_eq!(registry.len(), 1);
assert!(registry.contains("panic-model"));
}
#[tokio::test]
async fn test_loading_lock_cleanup_on_early_return() {
let registry = Arc::new(TokenizerRegistry::new());
// Load a tokenizer
let id1 = TokenizerRegistry::generate_id();
registry
.load(&id1, "model1", "source1", || async {
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
})
.await
.unwrap();
// Now simulate concurrent load attempts where one thread wins
// and another thread sees "already exists" in the double-check.
// The RAII guard should clean up the lock on early return.
// First, verify loading_locks is empty after successful load
// by checking that we can load a different model without issues
let id2 = TokenizerRegistry::generate_id();
let outcome = registry
.load(&id2, "model2", "source2", || async {
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
})
.await
.unwrap();
assert!(outcome.is_newly_loaded());
assert_eq!(registry.len(), 2);
// Try to load model1 again - should return AlreadyExists
// and the lock should be cleaned up (not leak)
let id3 = TokenizerRegistry::generate_id();
let outcome = registry
.load(&id3, "model1", "source1", || async {
panic!("Loader should not be called for existing model");
})
.await
.unwrap();
assert!(!outcome.is_newly_loaded());
assert_eq!(outcome.id(), id1); // Returns original ID
}
}