diff --git a/benchmark/gsm8k/README.md b/benchmark/gsm8k/README.md index c110f533c..67585d997 100644 --- a/benchmark/gsm8k/README.md +++ b/benchmark/gsm8k/README.md @@ -1,5 +1,15 @@ ## Run benchmark +### Using GSM8K Platinum + +GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag: + +``` +python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum +``` + +For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum + ### Benchmark sglang ``` python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 diff --git a/benchmark/gsm8k/bench_other.py b/benchmark/gsm8k/bench_other.py index 6dcb9ad7c..5ec781c70 100644 --- a/benchmark/gsm8k/bench_other.py +++ b/benchmark/gsm8k/bench_other.py @@ -7,6 +7,7 @@ import time from concurrent.futures import ThreadPoolExecutor import numpy as np +from datasets import load_dataset from tqdm import tqdm from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate @@ -45,9 +46,16 @@ def main(args): call_generate = get_call_generate(args) # Read data - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - filename = download_and_cache_file(url) - lines = list(read_jsonl(filename)) + if args.platinum: + print("Loading GSM8K Platinum dataset from HuggingFace...") + dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test") + lines = [ + {"question": item["question"], "answer": item["answer"]} for item in dataset + ] + else: + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) # Construct prompts num_questions = args.num_questions @@ -128,7 +136,7 @@ def main(args): with open(args.result_file, "a") as fout: value = { - "task": "gsm8k", + "task": "gsm8k-platinum" if args.platinum else "gsm8k", "backend": args.backend, "num_gpus": 1, "latency": round(latency, 3), @@ -147,5 +155,10 @@ if __name__ == "__main__": parser.add_argument("--num-shots", type=int, default=5) parser.add_argument("--data-path", type=str, default="test.jsonl") parser.add_argument("--num-questions", type=int, default=200) + parser.add_argument( + "--platinum", + action="store_true", + help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)", + ) args = add_common_other_args_and_parse(parser) main(args) diff --git a/benchmark/gsm8k/bench_sglang.py b/benchmark/gsm8k/bench_sglang.py index 9cdc2cf84..98c28b39b 100644 --- a/benchmark/gsm8k/bench_sglang.py +++ b/benchmark/gsm8k/bench_sglang.py @@ -6,6 +6,7 @@ import re import time import numpy as np +from datasets import load_dataset from sglang.lang.api import set_default_backend from sglang.test.test_utils import ( @@ -48,11 +49,18 @@ def main(args): set_default_backend(select_sglang_backend(args)) # Read data - data_path = args.data_path - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - if not os.path.isfile(data_path): - data_path = download_and_cache_file(url) - lines = list(read_jsonl(data_path)) + if args.platinum: + print("Loading GSM8K Platinum dataset from HuggingFace...") + dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test") + lines = [ + {"question": item["question"], "answer": item["answer"]} for item in dataset + ] + else: + data_path = args.data_path + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) # Construct prompts num_questions = args.num_questions @@ -125,7 +133,7 @@ def main(args): with open(args.result_file, "a") as fout: value = { - "task": "gsm8k", + "task": "gsm8k-platinum" if args.platinum else "gsm8k", "backend": args.backend, "num_gpus": 1, "latency": round(latency, 3), @@ -144,5 +152,10 @@ if __name__ == "__main__": parser.add_argument("--num-shots", type=int, default=5) parser.add_argument("--data-path", type=str, default="test.jsonl") parser.add_argument("--num-questions", type=int, default=200) + parser.add_argument( + "--platinum", + action="store_true", + help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)", + ) args = add_common_sglang_args_and_parse(parser) main(args)