Misc fixes for eagle (flush_cache, CPU overhead) (#3014)

2025-01-20 20:25:13 -08:00
parent d2571dd5c7
commit 287d07a669
11 changed files with 133 additions and 96 deletions
--- a/python/sglang/test/test_programs.py
+++ b/python/sglang/test/test_programs.py
@@ -535,7 +535,8 @@ def test_hellaswag_select():

    # Compute accuracy
    accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels))
-    assert np.abs(accuracy_gen - accuracy) < 0.1
+    print(f"{accuracy=}, {accuracy_gen=}")
+    assert np.abs(accuracy_gen - accuracy) < 0.05
    assert np.abs(latency_gen - latency) < 1

    return accuracy, latency