chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[2/2] Support deterministic inference for temperature > 0 (#10678)

Browse Source 
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
Co-authored-by: hebiao064 <hebiaobuaa@gmail.com>
This commit is contained in:
Qiaolin Yu
2025-09-21 19:36:08 -07:00
committed by GitHub
parent 86527a4799
commit e2ac7888b8
12 changed files with 117 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/test/test_deterministic.py
View File

@@ -29,6 +29,7 @@ class BenchArgs:
port: int = 30000
batch_size: int = 1
temperature: float = 0.0
sampling_seed: int = None
max_new_tokens: int = 100
frequency_penalty: float = 0.0
presence_penalty: float = 0.0
@@ -45,6 +46,9 @@ class BenchArgs:
parser.add_argument("--port", type=int, default=BenchArgs.port)
parser.add_argument("--n-trials", type=int, default=50)
parser.add_argument("--temperature", type=float, default=BenchArgs.temperature)
parser.add_argument(
"--sampling-seed", type=int, default=BenchArgs.sampling_seed
)
parser.add_argument(
"--max-new-tokens", type=int, default=BenchArgs.max_new_tokens
)
@@ -92,6 +96,7 @@ def send_single(
"max_new_tokens": args.max_new_tokens,
"frequency_penalty": args.frequency_penalty,
"presence_penalty": args.presence_penalty,
"sampling_seed": args.sampling_seed,
},
"return_logprob": args.return_logprob,
"stream": args.stream,
@@ -140,6 +145,7 @@ def send_mixed(args, batch_size: int):
"max_new_tokens": args.max_new_tokens,
"frequency_penalty": args.frequency_penalty,
"presence_penalty": args.presence_penalty,
"sampling_seed": args.sampling_seed,
},
"return_logprob": args.return_logprob,
"stream": args.stream,
@@ -186,6 +192,7 @@ def send_prefix(args, batch_size: int, prompts: List[str]):
"max_new_tokens": args.max_new_tokens,
"frequency_penalty": args.frequency_penalty,
"presence_penalty": args.presence_penalty,
"sampling_seed": args.sampling_seed,
},
"return_logprob": args.return_logprob,
"stream": args.stream,