[Perf] Optimize radix tree for cache-aware load balancin (#14758)
This commit is contained in:
@@ -59,7 +59,14 @@
|
||||
during the next eviction cycle.
|
||||
*/
|
||||
|
||||
use std::{sync::Arc, thread, time::Duration};
|
||||
use std::{
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use dashmap::DashMap;
|
||||
use rand::Rng;
|
||||
@@ -77,7 +84,10 @@ use crate::{core::Worker, observability::metrics::RouterMetrics};
|
||||
pub struct CacheAwarePolicy {
|
||||
config: CacheAwareConfig,
|
||||
trees: Arc<DashMap<String, Arc<Tree>>>,
|
||||
/// Handle to the background eviction thread
|
||||
eviction_handle: Option<thread::JoinHandle<()>>,
|
||||
/// Flag to signal the eviction thread to stop
|
||||
shutdown_flag: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl CacheAwarePolicy {
|
||||
@@ -87,25 +97,48 @@ impl CacheAwarePolicy {
|
||||
|
||||
pub fn with_config(config: CacheAwareConfig) -> Self {
|
||||
let trees = Arc::new(DashMap::<String, Arc<Tree>>::new());
|
||||
let shutdown_flag = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Start background eviction thread if configured
|
||||
let eviction_handle = if config.eviction_interval_secs > 0 {
|
||||
let trees_clone = Arc::clone(&trees);
|
||||
let shutdown_clone = Arc::clone(&shutdown_flag);
|
||||
let max_tree_size = config.max_tree_size;
|
||||
let interval = config.eviction_interval_secs;
|
||||
|
||||
Some(thread::spawn(move || loop {
|
||||
thread::sleep(Duration::from_secs(interval));
|
||||
Some(thread::spawn(move || {
|
||||
// Use smaller sleep intervals to check shutdown flag more frequently
|
||||
let check_interval_ms = 100; // Check every 100ms
|
||||
let total_sleep_ms = interval * 1000;
|
||||
|
||||
// Evict for all model trees
|
||||
for tree_ref in trees_clone.iter() {
|
||||
let model_id = tree_ref.key();
|
||||
let tree = tree_ref.value();
|
||||
tree.evict_tenant_by_size(max_tree_size);
|
||||
debug!(
|
||||
"Cache eviction completed for model {}, max_size: {}",
|
||||
model_id, max_tree_size
|
||||
);
|
||||
loop {
|
||||
// Sleep in small increments, checking shutdown flag periodically
|
||||
let mut slept_ms = 0u64;
|
||||
while slept_ms < total_sleep_ms {
|
||||
if shutdown_clone.load(Ordering::Relaxed) {
|
||||
debug!("Eviction thread received shutdown signal");
|
||||
return;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(check_interval_ms));
|
||||
slept_ms += check_interval_ms;
|
||||
}
|
||||
|
||||
// Check shutdown before starting eviction
|
||||
if shutdown_clone.load(Ordering::Relaxed) {
|
||||
debug!("Eviction thread received shutdown signal");
|
||||
return;
|
||||
}
|
||||
|
||||
// Evict for all model trees
|
||||
for tree_ref in trees_clone.iter() {
|
||||
let model_id = tree_ref.key();
|
||||
let tree = tree_ref.value();
|
||||
tree.evict_tenant_by_size(max_tree_size);
|
||||
debug!(
|
||||
"Cache eviction completed for model {}, max_size: {}",
|
||||
model_id, max_tree_size
|
||||
);
|
||||
}
|
||||
}
|
||||
}))
|
||||
} else {
|
||||
@@ -116,6 +149,7 @@ impl CacheAwarePolicy {
|
||||
config,
|
||||
trees,
|
||||
eviction_handle,
|
||||
shutdown_flag,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,12 +441,16 @@ impl Default for CacheAwarePolicy {
|
||||
|
||||
impl Drop for CacheAwarePolicy {
|
||||
fn drop(&mut self) {
|
||||
// Note: We can't properly stop the eviction thread since it's in an infinite loop
|
||||
// In a production system, we'd use a channel or atomic flag to signal shutdown
|
||||
// Signal the eviction thread to stop
|
||||
self.shutdown_flag.store(true, Ordering::Relaxed);
|
||||
|
||||
// Wait for the thread to finish (with timeout)
|
||||
if let Some(handle) = self.eviction_handle.take() {
|
||||
// The thread will continue running until the program exits
|
||||
// This is acceptable for now since the router typically runs for the lifetime of the program
|
||||
drop(handle);
|
||||
// The thread checks the shutdown flag every 100ms, so it should exit quickly
|
||||
match handle.join() {
|
||||
Ok(()) => debug!("Eviction thread shut down cleanly"),
|
||||
Err(_) => debug!("Eviction thread panicked during shutdown"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user