Enable CPU device on SGLang (#2806)

This commit is contained in:
Chunyuan WU
2025-01-16 21:22:53 -08:00
committed by GitHub
parent a8ccacc8b8
commit 63051738a9
13 changed files with 376 additions and 9 deletions
+2
View File
@@ -317,6 +317,8 @@ class Scheduler:
self.last_decode_stats_tic = time.time()
self.stream_interval = server_args.stream_interval
self.current_stream = torch.get_device_module(self.device).current_stream()
if self.device == "cpu":
self.current_stream.synchronize = lambda: None # No-op for CPU
# Session info
self.sessions: Dict[str, Session] = {}