[model-gateway] Refine TokenizerRegisty.load() to handle duplication (#17230)
This commit is contained in:
@@ -27,7 +27,7 @@ mod tests;
|
||||
use factory::{create_tokenizer_from_file, create_tokenizer_with_chat_template};
|
||||
// Re-export types used outside this module
|
||||
pub use huggingface::HuggingFaceTokenizer;
|
||||
pub use registry::TokenizerRegistry;
|
||||
pub use registry::{LoadError, LoadOutcome, TokenizerRegistry};
|
||||
pub use stop::StopSequenceDecoder;
|
||||
pub use stream::DecodeStream;
|
||||
pub use traits::{Decoder, Encoder, Encoding, SpecialTokens};
|
||||
|
||||
@@ -19,12 +19,51 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{debug, info};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::traits::Tokenizer;
|
||||
|
||||
/// Outcome of a tokenizer load operation
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum LoadOutcome {
|
||||
/// Tokenizer was newly loaded and registered
|
||||
Loaded { id: String },
|
||||
/// Tokenizer already existed, returning existing ID
|
||||
AlreadyExists { id: String },
|
||||
}
|
||||
|
||||
impl LoadOutcome {
|
||||
/// Get the ID regardless of outcome
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
LoadOutcome::Loaded { id } => id,
|
||||
LoadOutcome::AlreadyExists { id } => id,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the tokenizer was newly loaded
|
||||
pub fn is_newly_loaded(&self) -> bool {
|
||||
matches!(self, LoadOutcome::Loaded { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type for tokenizer loading operations
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LoadError {
|
||||
/// Name cannot be empty
|
||||
#[error("tokenizer name cannot be empty")]
|
||||
EmptyName,
|
||||
/// Source cannot be empty
|
||||
#[error("tokenizer source cannot be empty")]
|
||||
EmptySource,
|
||||
/// Loading failed
|
||||
#[error("{0}")]
|
||||
LoadFailed(String),
|
||||
}
|
||||
|
||||
/// Metadata and tokenizer instance for a registered tokenizer
|
||||
#[derive(Clone)]
|
||||
pub struct TokenizerEntry {
|
||||
@@ -53,6 +92,19 @@ pub struct TokenizerRegistry {
|
||||
loading_locks: DashMap<String, Arc<Mutex<()>>>,
|
||||
}
|
||||
|
||||
/// RAII guard that removes the loading lock entry on drop.
|
||||
/// Ensures cleanup happens on normal completion, early return, or panic.
|
||||
struct LoadingLockGuard<'a> {
|
||||
locks: &'a DashMap<String, Arc<Mutex<()>>>,
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl Drop for LoadingLockGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.locks.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenizerRegistry {
|
||||
/// Create a new empty registry
|
||||
pub fn new() -> Self {
|
||||
@@ -68,36 +120,46 @@ impl TokenizerRegistry {
|
||||
Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
/// Load and register a tokenizer with a pre-generated ID
|
||||
/// Load and register a tokenizer
|
||||
///
|
||||
/// If the tokenizer is already loaded (by name), returns the existing ID.
|
||||
/// Otherwise, uses the provided loader function to load it.
|
||||
/// Validates inputs, handles deduplication, and loads the tokenizer if needed.
|
||||
/// Per-key locking ensures only one load happens per name, preventing race conditions.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `id` - Pre-generated UUID for this tokenizer
|
||||
/// * `name` - User-provided name
|
||||
/// * `source` - Source path or HuggingFace model ID
|
||||
/// * `id` - Pre-generated UUID (use `generate_id()` to create one)
|
||||
/// * `name` - User-provided name (used for deduplication, must not be empty)
|
||||
/// * `source` - Source path or HuggingFace model ID (must not be empty)
|
||||
/// * `loader` - Async function that loads the tokenizer
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(id)` - Successfully loaded and registered (returns the ID)
|
||||
/// * `Err(message)` - Error message if loading fails
|
||||
/// * `Ok(LoadOutcome::Loaded { id })` - Tokenizer was newly loaded
|
||||
/// * `Ok(LoadOutcome::AlreadyExists { id })` - Tokenizer already existed
|
||||
/// * `Err(LoadError)` - Validation failed or loading failed
|
||||
pub async fn load<F, Fut>(
|
||||
&self,
|
||||
id: &str,
|
||||
name: &str,
|
||||
source: &str,
|
||||
loader: F,
|
||||
) -> Result<String, String>
|
||||
) -> Result<LoadOutcome, LoadError>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<Arc<dyn Tokenizer>, String>>,
|
||||
{
|
||||
// Validate inputs
|
||||
if name.is_empty() {
|
||||
return Err(LoadError::EmptyName);
|
||||
}
|
||||
if source.is_empty() {
|
||||
return Err(LoadError::EmptySource);
|
||||
}
|
||||
|
||||
// Fast path: already loaded by name
|
||||
if let Some(existing_id) = self.name_to_id.get(name) {
|
||||
debug!("Tokenizer already registered for name: {}", name);
|
||||
return Ok(existing_id.clone());
|
||||
return Ok(LoadOutcome::AlreadyExists {
|
||||
id: existing_id.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
debug!("Tokenizer cache miss for name: {}", name);
|
||||
@@ -109,24 +171,27 @@ impl TokenizerRegistry {
|
||||
.or_insert_with(|| Arc::new(Mutex::new(())))
|
||||
.clone();
|
||||
|
||||
let _guard = lock.lock().await;
|
||||
let _mutex_guard = lock.lock().await;
|
||||
let _lock_cleanup = LoadingLockGuard {
|
||||
locks: &self.loading_locks,
|
||||
key: name.to_string(),
|
||||
};
|
||||
|
||||
// Double-check after acquiring lock (another thread may have loaded it)
|
||||
if let Some(existing_id) = self.name_to_id.get(name) {
|
||||
debug!("Tokenizer loaded by another thread for name: {}", name);
|
||||
return Ok(existing_id.clone());
|
||||
return Ok(LoadOutcome::AlreadyExists {
|
||||
id: existing_id.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
// Load tokenizer
|
||||
info!("Loading tokenizer '{}' from source: {}", name, source);
|
||||
let result = loader().await;
|
||||
|
||||
// Always clean up the lock, whether loading succeeded or failed
|
||||
self.loading_locks.remove(name);
|
||||
let tokenizer = result.map_err(LoadError::LoadFailed)?;
|
||||
|
||||
let tokenizer = result?;
|
||||
|
||||
// Create entry
|
||||
// Create entry with provided ID
|
||||
let entry = TokenizerEntry {
|
||||
id: id.to_string(),
|
||||
name: name.to_string(),
|
||||
@@ -143,18 +208,21 @@ impl TokenizerRegistry {
|
||||
name, id
|
||||
);
|
||||
|
||||
Ok(id.to_string())
|
||||
Ok(LoadOutcome::Loaded { id: id.to_string() })
|
||||
}
|
||||
|
||||
/// Register a pre-loaded tokenizer with a pre-generated ID
|
||||
/// Register a preloaded tokenizer with a pre-generated ID
|
||||
///
|
||||
/// Atomically inserts a tokenizer into the registry only if no tokenizer
|
||||
/// with the same name exists. Returns the ID if successful.
|
||||
///
|
||||
/// This is primarily used for testing. Production code should use `load()`.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Some(id)` - If the tokenizer was successfully registered
|
||||
/// * `None` - If a tokenizer with this name already existed
|
||||
pub fn register(
|
||||
#[cfg(test)]
|
||||
pub(crate) fn register(
|
||||
&self,
|
||||
id: &str,
|
||||
name: &str,
|
||||
@@ -292,13 +360,17 @@ mod tests {
|
||||
|
||||
// Load and register a tokenizer
|
||||
let id = TokenizerRegistry::generate_id();
|
||||
registry
|
||||
let outcome = registry
|
||||
.load(&id, "model1", "path/to/model", || async {
|
||||
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify LoadOutcome::Loaded
|
||||
assert!(outcome.is_newly_loaded());
|
||||
assert_eq!(outcome.id(), id);
|
||||
|
||||
// Verify it's loaded
|
||||
assert!(!registry.is_empty());
|
||||
assert_eq!(registry.len(), 1);
|
||||
@@ -317,6 +389,65 @@ mod tests {
|
||||
assert!(registry.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_returns_already_exists() {
|
||||
let registry = TokenizerRegistry::new();
|
||||
let id1 = TokenizerRegistry::generate_id();
|
||||
let id2 = TokenizerRegistry::generate_id();
|
||||
|
||||
// First load should return Loaded
|
||||
let outcome1 = registry
|
||||
.load(&id1, "model1", "source1", || async {
|
||||
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(outcome1.is_newly_loaded());
|
||||
assert_eq!(outcome1.id(), id1);
|
||||
|
||||
// Second load with same name should return AlreadyExists with ORIGINAL id
|
||||
let outcome2 = registry
|
||||
.load(&id2, "model1", "source2", || async {
|
||||
panic!("Loader should not be called for duplicate name");
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!outcome2.is_newly_loaded());
|
||||
assert_eq!(outcome2.id(), id1); // Returns the original ID, not id2
|
||||
|
||||
// Registry still has only one entry
|
||||
assert_eq!(registry.len(), 1);
|
||||
|
||||
// Original source is preserved
|
||||
let entry = registry.get_by_name("model1").unwrap();
|
||||
assert_eq!(entry.source, "source1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_validation() {
|
||||
let registry = TokenizerRegistry::new();
|
||||
let id = TokenizerRegistry::generate_id();
|
||||
|
||||
// Empty name should fail
|
||||
let result = registry
|
||||
.load(&id, "", "source", || async {
|
||||
panic!("Loader should not be called for invalid input");
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(LoadError::EmptyName)));
|
||||
|
||||
// Empty source should fail
|
||||
let result = registry
|
||||
.load(&id, "model", "", || async {
|
||||
panic!("Loader should not be called for invalid input");
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(LoadError::EmptySource)));
|
||||
|
||||
// Registry should be empty (nothing was loaded)
|
||||
assert!(registry.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_prevents_duplicate_loading() {
|
||||
let registry = Arc::new(TokenizerRegistry::new());
|
||||
@@ -459,4 +590,88 @@ mod tests {
|
||||
assert!(result3.is_some());
|
||||
assert_eq!(registry.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_loading_lock_cleanup_on_panic() {
|
||||
let registry = Arc::new(TokenizerRegistry::new());
|
||||
|
||||
// Spawn a task that will panic during loading
|
||||
let registry_clone = registry.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
registry_clone
|
||||
.load(
|
||||
&TokenizerRegistry::generate_id(),
|
||||
"panic-model",
|
||||
"source",
|
||||
|| async {
|
||||
panic!("Simulated panic during tokenizer loading");
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
// Wait for the task - it should panic
|
||||
let result = handle.await;
|
||||
assert!(result.is_err(), "Task should have panicked");
|
||||
|
||||
// The RAII guard should have cleaned up the loading lock.
|
||||
// Verify by attempting another load with the same name - it should work,
|
||||
// not deadlock or fail due to stale lock.
|
||||
let id = TokenizerRegistry::generate_id();
|
||||
let outcome = registry
|
||||
.load(&id, "panic-model", "source", || async {
|
||||
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
|
||||
})
|
||||
.await;
|
||||
|
||||
// Should succeed - the lock was properly cleaned up
|
||||
assert!(outcome.is_ok(), "Load should succeed after panic cleanup");
|
||||
assert!(outcome.unwrap().is_newly_loaded());
|
||||
assert_eq!(registry.len(), 1);
|
||||
assert!(registry.contains("panic-model"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_loading_lock_cleanup_on_early_return() {
|
||||
let registry = Arc::new(TokenizerRegistry::new());
|
||||
|
||||
// Load a tokenizer
|
||||
let id1 = TokenizerRegistry::generate_id();
|
||||
registry
|
||||
.load(&id1, "model1", "source1", || async {
|
||||
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Now simulate concurrent load attempts where one thread wins
|
||||
// and another thread sees "already exists" in the double-check.
|
||||
// The RAII guard should clean up the lock on early return.
|
||||
|
||||
// First, verify loading_locks is empty after successful load
|
||||
// by checking that we can load a different model without issues
|
||||
let id2 = TokenizerRegistry::generate_id();
|
||||
let outcome = registry
|
||||
.load(&id2, "model2", "source2", || async {
|
||||
Ok(Arc::new(MockTokenizer::default()) as Arc<dyn Tokenizer>)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(outcome.is_newly_loaded());
|
||||
assert_eq!(registry.len(), 2);
|
||||
|
||||
// Try to load model1 again - should return AlreadyExists
|
||||
// and the lock should be cleaned up (not leak)
|
||||
let id3 = TokenizerRegistry::generate_id();
|
||||
let outcome = registry
|
||||
.load(&id3, "model1", "source1", || async {
|
||||
panic!("Loader should not be called for existing model");
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!outcome.is_newly_loaded());
|
||||
assert_eq!(outcome.id(), id1); // Returns original ID
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user