From 7f5879985c37c918e3fe33f256ab1d63365a0c4f Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Mon, 29 Dec 2025 08:37:24 -0800 Subject: [PATCH] [model-gateway] Improve tree benchmark with realistic multi-tenant scenarios (#14838) --- .github/workflows/pr-benchmark-rust.yml | 103 +- sgl-model-gateway/benches/tree_benchmark.rs | 1124 ++++++++++++++++--- 2 files changed, 1067 insertions(+), 160 deletions(-) diff --git a/.github/workflows/pr-benchmark-rust.yml b/.github/workflows/pr-benchmark-rust.yml index 99f5442fd..577c53155 100644 --- a/.github/workflows/pr-benchmark-rust.yml +++ b/.github/workflows/pr-benchmark-rust.yml @@ -281,9 +281,85 @@ jobs: if: always() run: sccache --show-stats + benchmark-tree: + name: Tree Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'router-benchmark') && + contains(github.event.pull_request.labels.*.name, 'run-ci'))) + # Use 4-core runner for concurrent benchmarks that spawn multiple threads + runs-on: 4-gpu-a10 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-model-gateway + # Share cache across all benchmark jobs + shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true + # Save cache even on failure + save-if: true + + - name: Run tree benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-model-gateway/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + # Run summary benchmark for quick validation in CI + # Note: Don't use --exact because benchmark names are prefixed (benchmark_summary/insert_realistic) + # Capture output to file for CI summary + cargo bench --bench tree_benchmark -- benchmark_summary 2>&1 | tee benchmark_output.txt + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tree-results-${{ github.sha }} + path: | + sgl-model-gateway/target/criterion/benchmark_summary*/ + sgl-model-gateway/benchmark_output.txt + retention-days: 30 + + - name: Show sccache stats + if: always() + run: sccache --show-stats + benchmark-summary: name: Benchmark Summary - needs: [benchmark-request-processing, benchmark-tokenizer, benchmark-tool-parser] + needs: [benchmark-request-processing, benchmark-tokenizer, benchmark-tool-parser, benchmark-tree] if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') runs-on: ubuntu-latest steps: @@ -359,6 +435,31 @@ jobs: else echo "❌ Failed or skipped" >> summary.md fi + echo "" >> summary.md + + # Tree Benchmark + echo "### Radix Tree (Cache-Aware Routing)" >> summary.md + TREE_DIR="benchmark-results/tree-results-${{ github.sha }}" + if [ -d "$TREE_DIR" ]; then + echo "✅ **Completed**" >> summary.md + if [ -f "$TREE_DIR/benchmark_output.txt" ]; then + echo "" >> summary.md + echo "
" >> summary.md + echo "View Results" >> summary.md + echo "" >> summary.md + echo '```' >> summary.md + # Extract benchmark results (printed incrementally during run) + echo "=== Summary Results ===" >> summary.md + grep "\[BENCH_RESULT\] summary" "$TREE_DIR/benchmark_output.txt" | sed 's/\[BENCH_RESULT\] //' >> summary.md || true + echo "" >> summary.md + # Also show the full summary table if available + grep -A 100 "RADIX TREE BENCHMARK SUMMARY\|CI SUMMARY" "$TREE_DIR/benchmark_output.txt" | head -60 >> summary.md || cat "$TREE_DIR/benchmark_output.txt" | tail -60 >> summary.md + echo '```' >> summary.md + echo "
" >> summary.md + fi + else + echo "❌ Failed or skipped" >> summary.md + fi echo "" >> summary.md echo "---" >> summary.md diff --git a/sgl-model-gateway/benches/tree_benchmark.rs b/sgl-model-gateway/benches/tree_benchmark.rs index 2b1589cf0..8c908e720 100644 --- a/sgl-model-gateway/benches/tree_benchmark.rs +++ b/sgl-model-gateway/benches/tree_benchmark.rs @@ -1,17 +1,71 @@ //! Benchmarks for the radix tree implementation used in cache-aware routing. //! +//! This benchmark simulates realistic cache-aware routing scenarios with: +//! - Multiple tenants representing HTTP/gRPC endpoints (10 endpoints) +//! - High-pressure workloads with concurrent operations +//! - Realistic request text patterns (system prompts, user queries, etc.) +//! //! Run with: cargo bench --bench tree_benchmark +//! +//! For quick validation (CI): cargo bench --bench tree_benchmark -- benchmark_summary --exact -use std::{sync::Arc, thread}; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + thread, + time::Instant, +}; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use rand::{ distr::{Alphanumeric, SampleString}, - rng as thread_rng, + rng as thread_rng, Rng, }; // Import the tree module use sgl_model_gateway::policies::tree::Tree; +// Global results storage for summary +lazy_static::lazy_static! { + static ref BENCHMARK_RESULTS: Mutex> = Mutex::new(BTreeMap::new()); +} + +fn add_result(category: &str, result: String) { + let mut results = BENCHMARK_RESULTS.lock().unwrap(); + let index = results.len(); + let key = format!("{:03}_{}", index, category); + // Print result immediately so it's captured even if process is killed later + eprintln!("[BENCH_RESULT] {} | {}", category, result); + results.insert(key, result); +} + +/// Simulated HTTP/gRPC endpoints representing worker nodes +/// These mirror real-world deployment patterns with 10 tenants +const ENDPOINT_TENANTS: [&str; 10] = [ + "http://worker-0.sglang.svc.cluster.local:8000", + "http://worker-1.sglang.svc.cluster.local:8000", + "http://worker-2.sglang.svc.cluster.local:8000", + "http://worker-3.sglang.svc.cluster.local:8000", + "http://worker-4.sglang.svc.cluster.local:8000", + "grpc://worker-5.sglang.svc.cluster.local:50051", + "grpc://worker-6.sglang.svc.cluster.local:50051", + "grpc://worker-7.sglang.svc.cluster.local:50051", + "http://10.0.0.100:8000", + "http://10.0.0.101:8000", +]; + +/// Common conversation prefixes that create shared tree paths +const CONVERSATION_PREFIXES: [&str; 6] = [ + "<|system|>\nYou are a helpful assistant.\n<|user|>\n", + "<|im_start|>system\nYou are a helpful AI assistant.<|im_end|>\n<|im_start|>user\n", + "[INST] <>\nYou are a helpful assistant.\n<>\n\n", + "Human: ", + "User: ", + "### Instruction:\n", +]; + /// Generate random ASCII strings of given length fn random_ascii_string(len: usize) -> String { Alphanumeric.sample_string(&mut thread_rng(), len) @@ -24,11 +78,30 @@ fn random_prefixed_strings(prefix: &str, suffix_len: usize, count: usize) -> Vec .collect() } -/// Benchmark single-threaded insert throughput +/// Generate realistic LLM request texts with system prompts and user queries +fn generate_realistic_requests(count: usize) -> Vec { + let mut rng = thread_rng(); + (0..count) + .map(|_| { + let prefix_idx = rng.random_range(0..CONVERSATION_PREFIXES.len()); + // Realistic LLM request sizes: 1000-3000 chars (~250-750 tokens) + // This represents typical user queries with context + let query_len = rng.random_range(1000..3000); + format!( + "{}{}", + CONVERSATION_PREFIXES[prefix_idx], + random_ascii_string(query_len) + ) + }) + .collect() +} + +/// Benchmark single-threaded insert throughput with endpoint tenants fn bench_insert_throughput(c: &mut Criterion) { let mut group = c.benchmark_group("insert_throughput"); for text_len in [10, 50, 100, 500].iter() { + let printed = Arc::new(AtomicBool::new(false)); group.throughput(Throughput::Elements(1)); group.bench_with_input( BenchmarkId::new("random_text", text_len), @@ -37,16 +110,38 @@ fn bench_insert_throughput(c: &mut Criterion) { let tree = Tree::new(); let strings: Vec = (0..1000).map(|_| random_ascii_string(len)).collect(); let mut idx = 0; + let printed_clone = printed.clone(); - b.iter(|| { - tree.insert(black_box(&strings[idx % strings.len()]), "tenant1"); - idx += 1; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let tenant = ENDPOINT_TENANTS[idx % ENDPOINT_TENANTS.len()]; + tree.insert(black_box(&strings[idx % strings.len()]), tenant); + idx += 1; + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>10}", + format!("random_text_{}", len), + len, + ops_per_sec, + 10 + ); + add_result("insert", result); + printed_clone.store(true, Ordering::Relaxed); + } + + duration }); }, ); } - // Benchmark with shared prefixes (common cache scenario) + // Benchmark with shared prefixes (common cache scenario) - distributed across endpoints + let printed_prefix = Arc::new(AtomicBool::new(false)); group.bench_function("shared_prefix_100", |b| { let tree = Tree::new(); let prefixes = ["system:", "user:", "assistant:", "tool:"]; @@ -55,95 +150,817 @@ fn bench_insert_throughput(c: &mut Criterion) { .flat_map(|p| random_prefixed_strings(p, 50, 250)) .collect(); let mut idx = 0; + let printed = printed_prefix.clone(); - b.iter(|| { - tree.insert(black_box(&strings[idx % strings.len()]), "tenant1"); - idx += 1; + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let tenant = ENDPOINT_TENANTS[idx % ENDPOINT_TENANTS.len()]; + tree.insert(black_box(&strings[idx % strings.len()]), tenant); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>10}", + "shared_prefix", "~58", ops_per_sec, 10 + ); + add_result("insert", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark with realistic LLM request patterns + let printed_llm = Arc::new(AtomicBool::new(false)); + group.bench_function("realistic_llm_requests", |b| { + let tree = Tree::new(); + let requests = generate_realistic_requests(2000); + let mut idx = 0; + let printed = printed_llm.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let tenant = ENDPOINT_TENANTS[idx % ENDPOINT_TENANTS.len()]; + tree.insert(black_box(&requests[idx % requests.len()]), tenant); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>10}", + "realistic_llm", "~100", ops_per_sec, 10 + ); + add_result("insert", result); + printed.store(true, Ordering::Relaxed); + } + + duration }); }); group.finish(); } -/// Benchmark prefix_match latency +/// Benchmark prefix_match latency with multi-tenant tree fn bench_prefix_match_latency(c: &mut Criterion) { let mut group = c.benchmark_group("prefix_match_latency"); - // Setup: pre-populate tree with data - let tree = Tree::new(); + // Setup: pre-populate tree with data distributed across all endpoints + let tree = Arc::new(Tree::new()); let prefixes = ["system:", "user:", "assistant:", "tool:"]; let strings: Vec = prefixes .iter() .flat_map(|p| random_prefixed_strings(p, 50, 1000)) .collect(); - for s in &strings { - tree.insert(s, "tenant1"); + // Distribute entries across all 10 endpoint tenants + for (i, s) in strings.iter().enumerate() { + let tenant = ENDPOINT_TENANTS[i % ENDPOINT_TENANTS.len()]; + tree.insert(s, tenant); } // Benchmark cache hit (exact match) + let printed_hit = Arc::new(AtomicBool::new(false)); + let tree_clone = tree.clone(); + let strings_clone = strings.clone(); group.bench_function("cache_hit", |b| { let mut idx = 0; - b.iter(|| { - let result = tree.prefix_match(black_box(&strings[idx % strings.len()])); - idx += 1; - result + let printed = printed_hit.clone(); + let tree = tree_clone.clone(); + let strings = strings_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&strings[idx % strings.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let latency_ns = duration.as_nanos() as f64 / iters as f64; + let result = format!( + "{:<20} | {:>12.0} | {:>12.1}", + "cache_hit", ops_per_sec, latency_ns + ); + add_result("prefix_match", result); + printed.store(true, Ordering::Relaxed); + } + + duration }); }); // Benchmark cache miss (no match) let miss_strings: Vec = (0..1000).map(|_| random_ascii_string(50)).collect(); + let printed_miss = Arc::new(AtomicBool::new(false)); + let tree_clone = tree.clone(); group.bench_function("cache_miss", |b| { let mut idx = 0; - b.iter(|| { - let result = tree.prefix_match(black_box(&miss_strings[idx % miss_strings.len()])); - idx += 1; - result + let printed = printed_miss.clone(); + let tree = tree_clone.clone(); + let miss_strings = miss_strings.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&miss_strings[idx % miss_strings.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let latency_ns = duration.as_nanos() as f64 / iters as f64; + let result = format!( + "{:<20} | {:>12.0} | {:>12.1}", + "cache_miss", ops_per_sec, latency_ns + ); + add_result("prefix_match", result); + printed.store(true, Ordering::Relaxed); + } + + duration }); }); // Benchmark partial match + let printed_partial = Arc::new(AtomicBool::new(false)); + let tree_clone = tree.clone(); group.bench_function("partial_match", |b| { let partial_strings: Vec = prefixes .iter() .map(|p| format!("{}partial_query", p)) .collect(); let mut idx = 0; - b.iter(|| { - let result = - tree.prefix_match(black_box(&partial_strings[idx % partial_strings.len()])); - idx += 1; - result + let printed = printed_partial.clone(); + let tree = tree_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = + tree.prefix_match(black_box(&partial_strings[idx % partial_strings.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let latency_ns = duration.as_nanos() as f64 / iters as f64; + let result = format!( + "{:<20} | {:>12.0} | {:>12.1}", + "partial_match", ops_per_sec, latency_ns + ); + add_result("prefix_match", result); + printed.store(true, Ordering::Relaxed); + } + + duration }); }); group.finish(); } -/// Benchmark concurrent operations +/// Benchmark concurrent operations with high pressure (10 endpoint tenants) fn bench_concurrent_operations(c: &mut Criterion) { let mut group = c.benchmark_group("concurrent"); group.sample_size(50); // Reduce sample size for concurrent tests - // Mixed read/write workload - for num_threads in [2, 4, 8].iter() { + // Mixed read/write workload with endpoint-style tenants + for num_threads in [2, 4, 8, 16].iter() { + let printed = Arc::new(AtomicBool::new(false)); group.bench_with_input( BenchmarkId::new("mixed_workload", num_threads), num_threads, |b, &threads| { - b.iter(|| { + let printed_clone = printed.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let tree = Arc::new(Tree::new()); + let handles: Vec<_> = (0..threads) + .map(|t| { + let tree = Arc::clone(&tree); + thread::spawn(move || { + // Each thread uses a different endpoint tenant + let tenant = ENDPOINT_TENANTS[t % ENDPOINT_TENANTS.len()]; + for i in 0..200 { + let text = format!( + "{}thread{}_request{}", + CONVERSATION_PREFIXES[i % CONVERSATION_PREFIXES.len()], + t, + i + ); + if i % 3 == 0 { + tree.prefix_match(&text); + } else { + tree.insert(&text, tenant); + } + } + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = iters * threads as u64 * 200; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>12.0}", + format!("mixed_workload_{}_threads", threads), + threads, + ops_per_sec, + ops_per_sec / threads as f64 + ); + add_result("concurrent", result); + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + // High-contention scenario: all threads sharing same prefixes + let printed_contention = Arc::new(AtomicBool::new(false)); + group.bench_function("high_contention_10_tenants", |b| { + let printed = printed_contention.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let tree = Arc::new(Tree::new()); + let handles: Vec<_> = (0..10) + .map(|t| { + let tree = Arc::clone(&tree); + thread::spawn(move || { + let tenant = ENDPOINT_TENANTS[t]; + // All threads insert similar prefixes to create contention + for i in 0..100 { + let text = format!( + "<|system|>\nYou are a helpful assistant.\n<|user|>\nQuery {}", + i + ); + tree.insert(&text, tenant); + tree.prefix_match(&text); + } + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * 10 * 200; // 10 threads * 200 ops (100 inserts + 100 matches) + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>12.0}", + "high_contention", + 10, + ops_per_sec, + ops_per_sec / 10.0 + ); + add_result("concurrent", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +/// Benchmark eviction performance with multi-tenant scenarios +fn bench_eviction(c: &mut Criterion) { + let mut group = c.benchmark_group("eviction"); + group.sample_size(20); // Eviction is expensive + + for tree_size in [1000, 5000, 10000].iter() { + let printed = Arc::new(AtomicBool::new(false)); + group.bench_with_input( + BenchmarkId::new("evict_to_half_single_tenant", tree_size), + tree_size, + |b, &size| { + let printed_clone = printed.clone(); + + b.iter_custom(|iters| { + let mut total_duration = std::time::Duration::ZERO; + for _ in 0..iters { + // Setup: create tree with many entries for single tenant + let tree = Tree::new(); + let tenant = ENDPOINT_TENANTS[0]; + for i in 0..size { + tree.insert(&format!("entry_{:05}", i), tenant); + } + + let start = Instant::now(); + tree.evict_tenant_by_size(size / 2); + total_duration += start.elapsed(); + } + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / total_duration.as_secs_f64(); + let latency_ms = total_duration.as_millis() as f64 / iters as f64; + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>12.2}", + format!("single_tenant_{}", size), + size, + ops_per_sec, + latency_ms + ); + add_result("eviction", result); + printed_clone.store(true, Ordering::Relaxed); + } + + total_duration + }); + }, + ); + } + + // Multi-tenant eviction: 10 tenants with overlapping data + for tree_size in [1000, 5000, 10000].iter() { + let printed = Arc::new(AtomicBool::new(false)); + group.bench_with_input( + BenchmarkId::new("evict_multi_tenant_10", tree_size), + tree_size, + |b, &size| { + let printed_clone = printed.clone(); + + b.iter_custom(|iters| { + let mut total_duration = std::time::Duration::ZERO; + for _ in 0..iters { + // Setup: create tree with entries distributed across 10 tenants + let tree = Tree::new(); + for i in 0..size { + let tenant = ENDPOINT_TENANTS[i % ENDPOINT_TENANTS.len()]; + // Use shared prefixes to create overlapping tree structure + let prefix = CONVERSATION_PREFIXES[i % CONVERSATION_PREFIXES.len()]; + tree.insert(&format!("{}entry_{:05}", prefix, i), tenant); + } + + let start = Instant::now(); + tree.evict_tenant_by_size(size / 20); + total_duration += start.elapsed(); + } + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / total_duration.as_secs_f64(); + let latency_ms = total_duration.as_millis() as f64 / iters as f64; + let result = format!( + "{:<25} | {:>8} | {:>12.0} | {:>12.2}", + format!("multi_tenant_{}", size), + size, + ops_per_sec, + latency_ms + ); + add_result("eviction", result); + printed_clone.store(true, Ordering::Relaxed); + } + + total_duration + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark UTF-8 handling vs ASCII with multiple endpoint tenants +fn bench_utf8_vs_ascii(c: &mut Criterion) { + let mut group = c.benchmark_group("encoding"); + + let tree_ascii = Arc::new(Tree::new()); + let tree_utf8 = Arc::new(Tree::new()); + + // Pre-populate with data distributed across endpoints + let ascii_strings: Vec = (0..1000).map(|_| random_ascii_string(50)).collect(); + let utf8_strings: Vec = (0..1000).map(|i| format!("你好世界_{}", i)).collect(); + + for (i, s) in ascii_strings.iter().enumerate() { + let tenant = ENDPOINT_TENANTS[i % ENDPOINT_TENANTS.len()]; + tree_ascii.insert(s, tenant); + } + for (i, s) in utf8_strings.iter().enumerate() { + let tenant = ENDPOINT_TENANTS[i % ENDPOINT_TENANTS.len()]; + tree_utf8.insert(s, tenant); + } + + let printed_ascii = Arc::new(AtomicBool::new(false)); + let tree_ascii_clone = tree_ascii.clone(); + let ascii_strings_clone = ascii_strings.clone(); + group.bench_function("ascii_match", |b| { + let mut idx = 0; + let printed = printed_ascii.clone(); + let tree = tree_ascii_clone.clone(); + let strings = ascii_strings_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&strings[idx % strings.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<20} | {:>12.0} | {:>12}", + "ASCII", ops_per_sec, "baseline" + ); + add_result("encoding", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + let printed_utf8 = Arc::new(AtomicBool::new(false)); + let tree_utf8_clone = tree_utf8.clone(); + let utf8_strings_clone = utf8_strings.clone(); + group.bench_function("utf8_match", |b| { + let mut idx = 0; + let printed = printed_utf8.clone(); + let tree = tree_utf8_clone.clone(); + let strings = utf8_strings_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&strings[idx % strings.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!("{:<20} | {:>12.0} | {:>12}", "UTF-8", ops_per_sec, "N/A"); + add_result("encoding", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +/// Benchmark multi-tenant scenarios with 10 HTTP/gRPC endpoint tenants +fn bench_multi_tenant(c: &mut Criterion) { + let mut group = c.benchmark_group("multi_tenant"); + + let tree = Arc::new(Tree::new()); + + // Setup: 10 endpoint tenants with overlapping data patterns + let prefixes = ["prompt:", "completion:", "context:", "system:", "user:"]; + + for tenant in &ENDPOINT_TENANTS { + for prefix in &prefixes { + for i in 0..200 { + tree.insert(&format!("{}data_{}", prefix, i), tenant); + } + } + } + + let printed_shared = Arc::new(AtomicBool::new(false)); + let tree_clone = tree.clone(); + group.bench_function("shared_prefix_lookup_10_tenants", |b| { + let queries: Vec = prefixes + .iter() + .flat_map(|p| (0..50).map(move |i| format!("{}data_{}", p, i))) + .collect(); + let mut idx = 0; + let printed = printed_shared.clone(); + let tree = tree_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&queries[idx % queries.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<30} | {:>10} | {:>12.0}", + "shared_prefix_lookup", 10, ops_per_sec + ); + add_result("multi_tenant", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + let printed_specific = Arc::new(AtomicBool::new(false)); + let tree_clone = tree.clone(); + group.bench_function("tenant_specific_match_10_tenants", |b| { + let queries: Vec<(String, &str)> = ENDPOINT_TENANTS + .iter() + .flat_map(|&t| (0..20).map(move |i| (format!("prompt:data_{}", i), t))) + .collect(); + let mut idx = 0; + let printed = printed_specific.clone(); + let tree = tree_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let (query, tenant) = &queries[idx % queries.len()]; + let result = tree.prefix_match_tenant(black_box(query), black_box(tenant)); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let result = format!( + "{:<30} | {:>10} | {:>12.0}", + "tenant_specific_match", 10, ops_per_sec + ); + add_result("multi_tenant", result); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark tenant removal (simulates worker going offline) + let printed_removal = Arc::new(AtomicBool::new(false)); + group.bench_function("tenant_removal", |b| { + let printed = printed_removal.clone(); + + b.iter_custom(|iters| { + let mut total_duration = std::time::Duration::ZERO; + for _ in 0..iters { + // Setup: create tree with all endpoints + let tree = Tree::new(); + for tenant in &ENDPOINT_TENANTS { + for prefix in &prefixes { + for i in 0..100 { + tree.insert(&format!("{}data_{}", prefix, i), tenant); + } + } + } + + let start = Instant::now(); + tree.remove_tenant(ENDPOINT_TENANTS[0]); + total_duration += start.elapsed(); + } + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / total_duration.as_secs_f64(); + let latency_ms = total_duration.as_millis() as f64 / iters as f64; + let result = format!( + "{:<30} | {:>10} | {:>12.0} | {:>10.2}ms", + "tenant_removal", 10, ops_per_sec, latency_ms + ); + add_result("multi_tenant", result); + printed.store(true, Ordering::Relaxed); + } + + total_duration + }); + }); + + group.finish(); +} + +/// Generate worker endpoint URLs for scaling tests +fn generate_worker_endpoints(count: usize) -> Vec { + (0..count) + .map(|i| { + if i % 4 == 0 { + format!("grpc://worker-{}.sglang.svc.cluster.local:50051", i) + } else { + format!("http://worker-{}.sglang.svc.cluster.local:8000", i) + } + }) + .collect() +} + +/// Benchmark summary for CI - runs a subset of representative benchmarks +fn bench_summary(c: &mut Criterion) { + let mut group = c.benchmark_group("benchmark_summary"); + + // Reduce warmup and measurement time for faster CI runs + group.warm_up_time(std::time::Duration::from_secs(1)); + group.measurement_time(std::time::Duration::from_secs(3)); + + // Configuration constants + const TREE_SIZE: usize = 10_000; // Realistic cache size + const INSERT_POOL_SIZE: usize = 10_000; // Unique requests for insert tests + const NUM_THREADS: usize = 8; + const OPS_PER_THREAD: usize = 100; + + // Worker scaling configurations to test + // Using 2 representative counts (low/high) to keep CI runtime reasonable + // while still demonstrating scaling behavior + const WORKER_COUNTS: [usize; 2] = [10, 100]; + + // Pre-generate requests for tree population and queries + let requests = generate_realistic_requests(TREE_SIZE); + let avg_len: usize = requests.iter().map(|r| r.len()).sum::() / requests.len(); + + // Report test configuration upfront + add_result("config", "Test Configuration:".to_string()); + add_result( + "config", + format!( + " Request size: ~{} chars (~{} tokens)", + avg_len, + avg_len / 4 + ), + ); + add_result( + "config", + format!(" Tree size: {} entries (for MATCH tests)", TREE_SIZE), + ); + add_result( + "config", + format!(" Insert pool: {} unique requests", INSERT_POOL_SIZE), + ); + add_result( + "config", + format!( + " Concurrency: {} threads x {} ops/thread", + NUM_THREADS, OPS_PER_THREAD + ), + ); + add_result( + "config", + format!(" Worker counts tested: {:?}", WORKER_COUNTS), + ); + + // Test INSERT performance at different worker scales + // Use large pool of unique requests to avoid measuring cache-hit behavior + let insert_requests = generate_realistic_requests(INSERT_POOL_SIZE); + + for &num_workers in &WORKER_COUNTS { + let workers = generate_worker_endpoints(num_workers); + let printed = Arc::new(AtomicBool::new(false)); + let bench_name = format!("insert_{}w", num_workers); + let insert_requests = insert_requests.clone(); + + group.bench_function(&bench_name, |b| { + let workers = workers.clone(); + let printed = printed.clone(); + let insert_requests = insert_requests.clone(); + + b.iter_custom(|iters| { + // Fresh tree for each measurement to test pure insert performance + let tree = Tree::new(); + + let start = Instant::now(); + for i in 0..iters { + let tenant = &workers[i as usize % workers.len()]; + // Use pre-generated unique requests from large pool + let text = &insert_requests[i as usize % insert_requests.len()]; + tree.insert(black_box(text), tenant); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let latency_us = duration.as_nanos() as f64 / iters as f64 / 1000.0; + let throughput_mb = (ops_per_sec * avg_len as f64) / 1_000_000.0; + add_result( + "summary", + format!( + "INSERT {:>3} workers: {:>8.0} ops/sec | {:>5.1} µs/op | {:>6.1} MB/s | ~{} chars", + num_workers, ops_per_sec, latency_us, throughput_mb, avg_len + ), + ); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + // Test MATCH performance at different worker scales + // Tree is pre-populated with TREE_SIZE entries distributed across workers + for &num_workers in &WORKER_COUNTS { + let workers = generate_worker_endpoints(num_workers); + let tree = Arc::new(Tree::new()); + + // Populate tree with requests distributed across workers + for (i, req) in requests.iter().enumerate() { + let tenant = &workers[i % workers.len()]; + tree.insert(req, tenant); + } + + let printed = Arc::new(AtomicBool::new(false)); + let requests_clone = requests.clone(); + let bench_name = format!("match_{}w", num_workers); + + group.bench_function(&bench_name, |b| { + let tree = tree.clone(); + let requests = requests_clone.clone(); + let mut idx = 0; + let printed = printed.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let result = tree.prefix_match(black_box(&requests[idx % requests.len()])); + black_box(result); + idx += 1; + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let latency_us = duration.as_nanos() as f64 / iters as f64 / 1000.0; + let throughput_mb = (ops_per_sec * avg_len as f64) / 1_000_000.0; + add_result( + "summary", + format!( + "MATCH {:>3} workers: {:>8.0} ops/sec | {:>5.1} µs/op | {:>6.1} MB/s | {}k tree entries", + num_workers, ops_per_sec, latency_us, throughput_mb, TREE_SIZE / 1000 + ), + ); + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + // Concurrent benchmark with scaling workers + // Reduced sample size and measurement time for CI + group.sample_size(10); + group.measurement_time(std::time::Duration::from_secs(3)); + for &num_workers in &WORKER_COUNTS { + let workers = generate_worker_endpoints(num_workers); + let printed = Arc::new(AtomicBool::new(false)); + let bench_name = format!("concurrent_{}w", num_workers); + + group.bench_function(&bench_name, |b| { + let printed = printed.clone(); + let workers = workers.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { let tree = Arc::new(Tree::new()); - let handles: Vec<_> = (0..threads) + let workers_ref = &workers; + let handles: Vec<_> = (0..NUM_THREADS) .map(|t| { let tree = Arc::clone(&tree); + let worker = workers_ref[t % workers_ref.len()].clone(); thread::spawn(move || { - let tenant = format!("tenant{}", t); - for i in 0..100 { - let text = format!("thread{}_request{}", t, i); + for i in 0..OPS_PER_THREAD { + let text = format!( + "{}thread{}_request{}", + CONVERSATION_PREFIXES[i % CONVERSATION_PREFIXES.len()], + t, + i + ); if i % 3 == 0 { tree.prefix_match(&text); } else { - tree.insert(&text, &tenant); + tree.insert(&text, &worker); } } }) @@ -153,142 +970,131 @@ fn bench_concurrent_operations(c: &mut Criterion) { for h in handles { h.join().unwrap(); } - }); - }, - ); - } + } + let duration = start.elapsed(); - group.finish(); -} + if !printed.load(Ordering::Relaxed) { + let total_ops = iters * NUM_THREADS as u64 * OPS_PER_THREAD as u64; + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let per_thread_ops = ops_per_sec / NUM_THREADS as f64; + add_result( + "summary", + format!( + "CONCURRENT {:>3} workers: {:>6.0} ops/sec | {} threads | {:.0} ops/thread", + num_workers, ops_per_sec, NUM_THREADS, per_thread_ops + ), + ); + printed.store(true, Ordering::Relaxed); + } -/// Benchmark eviction performance -fn bench_eviction(c: &mut Criterion) { - let mut group = c.benchmark_group("eviction"); - group.sample_size(20); // Eviction is expensive - - for tree_size in [1000, 5000, 10000].iter() { - group.bench_with_input( - BenchmarkId::new("evict_to_half", tree_size), - tree_size, - |b, &size| { - b.iter_with_setup( - || { - // Setup: create tree with many entries - let tree = Tree::new(); - for i in 0..size { - tree.insert(&format!("entry_{:05}", i), "tenant1"); - } - tree - }, - |tree| { - // Evict to half size - tree.evict_tenant_by_size(size / 2); - }, - ); - }, - ); - } - - group.finish(); -} - -/// Benchmark UTF-8 handling vs ASCII -fn bench_utf8_vs_ascii(c: &mut Criterion) { - let mut group = c.benchmark_group("encoding"); - - let tree_ascii = Tree::new(); - let tree_utf8 = Tree::new(); - - // Pre-populate - let ascii_strings: Vec = (0..1000).map(|_| random_ascii_string(50)).collect(); - let utf8_strings: Vec = (0..1000).map(|i| format!("你好世界_{}", i)).collect(); - - for s in &ascii_strings { - tree_ascii.insert(s, "tenant1"); - } - for s in &utf8_strings { - tree_utf8.insert(s, "tenant1"); - } - - group.bench_function("ascii_match", |b| { - let mut idx = 0; - b.iter(|| { - let result = - tree_ascii.prefix_match(black_box(&ascii_strings[idx % ascii_strings.len()])); - idx += 1; - result + duration + }); }); - }); - - group.bench_function("utf8_match", |b| { - let mut idx = 0; - b.iter(|| { - let result = tree_utf8.prefix_match(black_box(&utf8_strings[idx % utf8_strings.len()])); - idx += 1; - result - }); - }); + } group.finish(); } -/// Benchmark multi-tenant scenarios -fn bench_multi_tenant(c: &mut Criterion) { - let mut group = c.benchmark_group("multi_tenant"); +/// Print final summary table +fn print_summary() { + // Ensure output is flushed immediately + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); - let tree = Tree::new(); + eprintln!("\n{}", "=".repeat(100)); + eprintln!("RADIX TREE BENCHMARK SUMMARY (Cache-Aware Routing)"); + eprintln!("{}", "=".repeat(100)); - // Setup: multiple tenants with overlapping data - let tenants = ["worker1", "worker2", "worker3", "worker4"]; - let prefixes = ["prompt:", "completion:", "context:"]; + let results = BENCHMARK_RESULTS.lock().unwrap(); + eprintln!("Total benchmark results collected: {}", results.len()); - for tenant in &tenants { - for prefix in &prefixes { - for i in 0..100 { - tree.insert(&format!("{}data_{}", prefix, i), tenant); + let mut current_category = String::new(); + for (key, value) in results.iter() { + let category = key.split('_').skip(1).collect::>().join("_"); + + if category != current_category { + current_category = category.clone(); + + // Print section header based on category + eprintln!("\n{}", "-".repeat(100)); + match category.as_str() { + "insert" => { + eprintln!("INSERT THROUGHPUT (10 endpoint tenants)"); + eprintln!( + "{:<25} | {:>8} | {:>12} | {:>10}", + "Test Case", "Size", "Ops/sec", "Tenants" + ); + } + "prefix_match" => { + eprintln!("PREFIX MATCH LATENCY"); + eprintln!( + "{:<20} | {:>12} | {:>12}", + "Match Type", "Ops/sec", "Latency(ns)" + ); + } + "concurrent" => { + eprintln!("CONCURRENT OPERATIONS (mixed read/write)"); + eprintln!( + "{:<25} | {:>8} | {:>12} | {:>12}", + "Configuration", "Threads", "Total Ops/s", "Per-Thread" + ); + } + "eviction" => { + eprintln!("EVICTION PERFORMANCE"); + eprintln!( + "{:<25} | {:>8} | {:>12} | {:>12}", + "Configuration", "Size", "Ops/sec", "Latency(ms)" + ); + } + "encoding" => { + eprintln!("ENCODING (ASCII vs UTF-8)"); + eprintln!( + "{:<20} | {:>12} | {:>12}", + "Encoding", "Ops/sec", "Comparison" + ); + } + "multi_tenant" => { + eprintln!("MULTI-TENANT SCENARIOS (10 HTTP/gRPC endpoints)"); + eprintln!( + "{:<30} | {:>10} | {:>12}", + "Operation", "Tenants", "Ops/sec" + ); + } + "config" => { + eprintln!("TEST CONFIGURATION"); + } + "summary" => { + eprintln!("BENCHMARK RESULTS"); + } + _ => {} } + eprintln!("{}", "-".repeat(100)); } + + eprintln!("{}", value); } - group.bench_function("shared_prefix_lookup", |b| { - let queries: Vec = prefixes - .iter() - .flat_map(|p| (0..10).map(move |i| format!("{}data_{}", p, i))) - .collect(); - let mut idx = 0; - - b.iter(|| { - let result = tree.prefix_match(black_box(&queries[idx % queries.len()])); - idx += 1; - result - }); - }); - - group.bench_function("tenant_specific_match", |b| { - let queries: Vec<(String, &str)> = tenants - .iter() - .flat_map(|&t| (0..10).map(move |i| (format!("prompt:data_{}", i), t))) - .collect(); - let mut idx = 0; - - b.iter(|| { - let (query, tenant) = &queries[idx % queries.len()]; - let result = tree.prefix_match_tenant(black_box(query), black_box(tenant)); - idx += 1; - result - }); - }); - - group.finish(); + eprintln!("\n{}", "=".repeat(100)); + eprintln!("Endpoint tenants used:"); + for (i, tenant) in ENDPOINT_TENANTS.iter().enumerate() { + eprintln!(" [{}] {}", i, tenant); + } + eprintln!("{}", "=".repeat(100)); } -criterion_group!( - benches, - bench_insert_throughput, - bench_prefix_match_latency, - bench_concurrent_operations, - bench_eviction, - bench_utf8_vs_ascii, - bench_multi_tenant, -); +fn run_benchmarks(c: &mut Criterion) { + bench_insert_throughput(c); + bench_prefix_match_latency(c); + bench_concurrent_operations(c); + bench_eviction(c); + bench_utf8_vs_ascii(c); + bench_multi_tenant(c); + bench_summary(c); + + // Print summary at the end + print_summary(); +} + +criterion_group!(benches, run_benchmarks); criterion_main!(benches);