feat(disagg): per-layer transfer manager (lever A, A2 orchestration)
Add PerLayerTransferManager: owns a worker thread-pool + the active per-request contexts for the current forward batch, registered as a KV-pool notifier. on_layer_end (forward thread) records ONE CUDA event on the compute stream and enqueues (ctx, layer, event) per active context; workers do the event-wait + async submit OFF the forward thread. finish(room) waits the request's transfers. event_factory/current_stream injected for unit-testability (no CUDA needed). Unit-tested (5 manager cases, 11 total in test_cp_per_layer_transfer.py): per-active-ctx enqueue with the event recorded on the stream, worker-step submit + mark-failed-on-exception, finish pop + idempotency, no-op when idle. The A3 scheduler wiring (notifier registration + setup-before-forward + finish-after + no-double-send reconcile) is the remaining hot-path step; plan locked in lever-a-implementation-plan.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -103,5 +103,107 @@ class TestPerLayerTransferContext(unittest.TestCase):
|
||||
self.assertEqual(ctx.finish(), -9)
|
||||
|
||||
|
||||
class _MockCtx:
|
||||
def __init__(self):
|
||||
self.submitted = []
|
||||
self.finished = False
|
||||
self.failed = False
|
||||
|
||||
def submit_layer(self, layer_id, event):
|
||||
self.submitted.append((layer_id, event))
|
||||
|
||||
def finish(self):
|
||||
self.finished = True
|
||||
return 0
|
||||
|
||||
def mark_failed(self):
|
||||
self.failed = True
|
||||
|
||||
|
||||
class _RecEvent:
|
||||
def __init__(self):
|
||||
self.recorded_stream = "UNSET"
|
||||
self.synced = False
|
||||
|
||||
def record(self, stream=None):
|
||||
self.recorded_stream = stream
|
||||
|
||||
def synchronize(self):
|
||||
self.synced = True
|
||||
|
||||
|
||||
def _manager(num_workers=0):
|
||||
from sglang.srt.disaggregation.cp_per_layer_transfer import PerLayerTransferManager
|
||||
|
||||
return PerLayerTransferManager(
|
||||
num_workers=num_workers, event_factory=_RecEvent, current_stream=lambda: "STREAM"
|
||||
)
|
||||
|
||||
|
||||
def _drain(q):
|
||||
import queue as _q
|
||||
|
||||
items = []
|
||||
while True:
|
||||
try:
|
||||
items.append(q.get(block=False))
|
||||
except _q.Empty:
|
||||
break
|
||||
return items
|
||||
|
||||
|
||||
class TestPerLayerTransferManager(unittest.TestCase):
|
||||
def test_on_layer_end_enqueues_per_active_ctx_with_recorded_event(self):
|
||||
m = _manager()
|
||||
c1, c2 = _MockCtx(), _MockCtx()
|
||||
m.register("r1", c1)
|
||||
m.register("r2", c2)
|
||||
m.on_layer_end(5)
|
||||
items = _drain(m._q)
|
||||
self.assertEqual(len(items), 2) # one per active context
|
||||
for ctx, layer_id, ev in items:
|
||||
self.assertEqual(layer_id, 5)
|
||||
self.assertEqual(ev.recorded_stream, "STREAM") # event recorded on compute stream
|
||||
|
||||
def test_worker_step_calls_submit_layer(self):
|
||||
m = _manager()
|
||||
c = _MockCtx()
|
||||
ev = _RecEvent()
|
||||
m._worker_step((c, 3, ev))
|
||||
self.assertEqual(c.submitted, [(3, ev)])
|
||||
|
||||
def test_worker_step_marks_failed_on_exception(self):
|
||||
m = _manager()
|
||||
|
||||
class _Boom:
|
||||
def __init__(self):
|
||||
self.failed = False
|
||||
|
||||
def submit_layer(self, layer_id, event):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def mark_failed(self):
|
||||
self.failed = True
|
||||
|
||||
b = _Boom()
|
||||
m._worker_step((b, 0, None))
|
||||
self.assertTrue(b.failed)
|
||||
|
||||
def test_finish_pops_and_calls_ctx_finish(self):
|
||||
m = _manager()
|
||||
c = _MockCtx()
|
||||
m.register("r1", c)
|
||||
self.assertTrue(m.has_active())
|
||||
self.assertEqual(m.finish("r1"), 0)
|
||||
self.assertTrue(c.finished)
|
||||
self.assertFalse(m.has_active())
|
||||
self.assertEqual(m.finish("r1"), 0) # idempotent after pop
|
||||
|
||||
def test_on_layer_end_no_active_is_noop(self):
|
||||
m = _manager()
|
||||
m.on_layer_end(0)
|
||||
self.assertEqual(_drain(m._q), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user