feat: TTL-based prefix pinning with refresh-on-hit for HiRadixCache (#18941)

Co-authored-by: Claude <noreply@anthropic.com>
2026-03-01 23:27:22 -08:00
parent e42fa009d4
commit f7da379b61
9 changed files with 296 additions and 7 deletions
@@ -277,6 +277,8 @@ class Envs:
    SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
    SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
    SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None)
+    # Max fraction of cache (by token count) that can be pinned; 0 = disable pinning.
+    SGLANG_HICACHE_MAX_PINNED_RATIO = EnvFloat(0.0)

    # Mooncake KV Transfer
    SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)