fix(disagg): cover notifier-missed layers via post-forward fallback (lever A)
E2E diagnostic was precise: "finish TIMEOUT processed=78/79", submit_failed=0 — the per-layer notifier fires 78x but kv_data_ptrs has 79 layers (the 79th is the MTP/nextn EAGLE buffer: present in kv_data_ptrs so the monolithic send moves it, but it doesn't fire the per-layer hook). The old completion required all num_layers, so it both hung to the timeout AND would silently miss that layer's KV (corruption). Redesign: gate completion on the ACTUAL enqueued count (note_enqueued), and in finish() SYNCHRONOUSLY transfer any layers the notifier didn't fire for (KV is fully written post-forward, no event needed). Net: the per-layer set == kv_data_ptrs, byte-identical to the monolithic send; robust to any model firing fewer hooks than KV buffers. The fired layers stay overlapped with the forward. Unit tests updated (25 pass): fallback transfers the missed layers; submit failures still report -1. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -102,24 +102,24 @@ class TestPerLayerTransferContext(unittest.TestCase):
|
||||
ctx.submit_layer(0, None)
|
||||
self.assertEqual(ctx.finish(), -9)
|
||||
|
||||
def test_finish_times_out_to_failure_when_layers_missing(self):
|
||||
# only 1 of 3 layers ever processed -> finish must NOT silently succeed
|
||||
def test_finish_falls_back_for_notifier_missed_layers(self):
|
||||
# only layer 0 fired by the notifier; finish must SYNCHRONOUSLY transfer the
|
||||
# missing layers 1,2 (e.g. an MTP buffer not hooked) -> all 3 moved, success.
|
||||
eng = _FakeEngine()
|
||||
ctx = _ctx(eng, num_layers=3)
|
||||
ctx.submit_layer(0, None)
|
||||
self.assertEqual(ctx.finish(timeout=0.05), -1) # incomplete -> failure
|
||||
ctx.submit_layer(0, None) # notifier fired layer 0 only
|
||||
self.assertEqual(ctx.finish(timeout=0.5), 0)
|
||||
self.assertEqual(len(eng.submits), 3) # layer 0 (overlap) + 1,2 (fallback)
|
||||
self.assertEqual(eng.waits[-1], [101, 102, 103])
|
||||
|
||||
def test_skipped_and_failed_layers_still_count_toward_completion(self):
|
||||
# layer 1 has no blocks (skip), layer 2 submit fails; all 3 still complete
|
||||
eng = _FakeEngine(submit_rets=[101, -1])
|
||||
|
||||
def blocks(L):
|
||||
return None if L == 1 else ([L], [L + 10], [8])
|
||||
|
||||
ctx = _ctx(eng, blocks, num_layers=3)
|
||||
def test_failed_submit_still_reported_after_fallback(self):
|
||||
# layer 2 submit fails during the overlap; finish reports -1 even though it
|
||||
# also runs the fallback for any unfired layers.
|
||||
eng = _FakeEngine(submit_rets=[101, -1, 103])
|
||||
ctx = _ctx(eng, num_layers=3)
|
||||
for L in range(3):
|
||||
ctx.submit_layer(L, None)
|
||||
self.assertEqual(ctx.finish(timeout=0.5), -1) # completes (no hang), reports failure
|
||||
self.assertEqual(ctx.finish(timeout=0.5), -1)
|
||||
|
||||
|
||||
class TestBuildLayerBlocks(unittest.TestCase):
|
||||
@@ -151,9 +151,13 @@ class TestBuildLayerBlocks(unittest.TestCase):
|
||||
class _MockCtx:
|
||||
def __init__(self):
|
||||
self.submitted = []
|
||||
self.enqueued = 0
|
||||
self.finished = False
|
||||
self.failed = False
|
||||
|
||||
def note_enqueued(self):
|
||||
self.enqueued += 1
|
||||
|
||||
def submit_layer(self, layer_id, event):
|
||||
self.submitted.append((layer_id, event))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user