691c8534cf
Co-authored-by: ryang-max <y1cunhui.yang@gmail.com> Co-authored-by: ryang <38470282+ryang-max@users.noreply.github.com>
11 lines
254 B
Python
11 lines
254 B
Python
# GPU Memory Types
|
|
GPU_MEMORY_TYPE_KV_CACHE = "kv_cache"
|
|
GPU_MEMORY_TYPE_WEIGHTS = "weights"
|
|
GPU_MEMORY_TYPE_CUDA_GRAPH = "cuda_graph"
|
|
|
|
GPU_MEMORY_ALL_TYPES = [
|
|
GPU_MEMORY_TYPE_KV_CACHE,
|
|
GPU_MEMORY_TYPE_WEIGHTS,
|
|
GPU_MEMORY_TYPE_CUDA_GRAPH,
|
|
]
|