diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py index 9d6536520..79a324302 100644 --- a/python/sglang/srt/layers/radix_attention.py +++ b/python/sglang/srt/layers/radix_attention.py @@ -142,8 +142,11 @@ def unified_attention_with_output( ret = forward_batch.attn_backend.forward( query, key, value, attention_layer, forward_batch, save_kv_cache ) - assert output.shape == ret.shape - output.copy_(ret) + assert ( + output.numel() == ret.numel() + ), f"Output tensor element mismatch: {output.numel()} != {ret.numel()}" + + output.view(ret.shape).copy_(ret) return diff --git a/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py b/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py index e4af95d17..7b36fd9cf 100644 --- a/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/piecewise_cuda_graph_runner.py @@ -262,9 +262,14 @@ class PiecewiseCudaGraphRunner: def can_run(self, forward_batch: ForwardBatch): num_tokens = len(forward_batch.input_ids) - # TODO(yuwei): support return logprob + # TODO(yuwei): support return input_ids' logprob if forward_batch.return_logprob: - return False + for start_len, seq_len in zip( + forward_batch.extend_logprob_start_lens_cpu, + forward_batch.extend_seq_lens_cpu, + ): + if start_len is not None and start_len < seq_len: + return False if num_tokens <= self.max_num_tokens: return True return False @@ -438,7 +443,7 @@ class PiecewiseCudaGraphRunner: out_cache_loc=out_cache_loc, seq_lens_sum=forward_batch.seq_lens_sum, encoder_lens=forward_batch.encoder_lens, - return_logprob=forward_batch.return_logprob, + return_logprob=False, extend_seq_lens=forward_batch.extend_seq_lens, extend_prefix_lens=forward_batch.extend_prefix_lens, extend_start_loc=forward_batch.extend_start_loc, diff --git a/test/srt/test_piecewise_cuda_graph.py b/test/srt/test_piecewise_cuda_graph.py index 4325d9acb..4b0042dbc 100644 --- a/test/srt/test_piecewise_cuda_graph.py +++ b/test/srt/test_piecewise_cuda_graph.py @@ -44,6 +44,18 @@ class TestPiecewiseCudaGraphCorrectness(CustomTestCase): metrics = run_eval(args) self.assertGreaterEqual(metrics["score"], 0.235) + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + class TestPiecewiseCudaGraphBenchmark(CustomTestCase):