Cleanup disagg decode prebuilt flow and add cross-stream sync in merge_batch (#19568)

Co-authored-by: vincent <vincent@vincentdeMacBook-Pro.local>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>

test/registered/distributed/test_disaggregation_dp_attention.py

@@ -125,6 +125,23 @@ class TestDisaggregationDPAttentionRoundRobin(TestDisaggregationDPAttention):
         self.assertLess(result["mean_tpot_ms"], 20)
         self.assertEqual(result["completed"], 1000)
 
+    def test_bench_serving_itl(self):
+        num_prompts = 512
+        args = get_benchmark_args(
+            base_url=f"http://{self.base_host}:{self.lb_port}",
+            dataset_name="random",
+            tokenizer=self.model,
+            num_prompts=num_prompts,
+            random_input_len=512,
+            random_output_len=64,
+            request_rate=float("inf"),
+            max_concurrency=64,
+        )
+        result = run_benchmark(args)
+
+        self.assertEqual(result["completed"], num_prompts)
+        self.assertLess(result["p99_itl_ms"], 20)
+
 
 @unittest.skip(
     "Skip this test until new testing logic in mini-lb has been updated in docker image."