[Docs] add v1/score api to native api documentation (#16568)
This commit is contained in:
@@ -17,6 +17,7 @@
|
||||
"- `/update_weights`\n",
|
||||
"- `/encode`(embedding model)\n",
|
||||
"- `/v1/rerank`(cross encoder rerank model)\n",
|
||||
"- `/v1/score`(decoder-only scoring)\n",
|
||||
"- `/classify`(reward model)\n",
|
||||
"- `/start_expert_distribution_record`\n",
|
||||
"- `/stop_expert_distribution_record`\n",
|
||||
@@ -129,7 +130,7 @@
|
||||
"Gets the server information including CLI arguments, token limits, and memory pool sizes.\n",
|
||||
"- Note: `get_server_info` merges the following deprecated endpoints:\n",
|
||||
" - `get_server_args`\n",
|
||||
" - `get_memory_pool_size` \n",
|
||||
" - `get_memory_pool_size`\n",
|
||||
" - `get_max_total_num_tokens`"
|
||||
]
|
||||
},
|
||||
@@ -366,6 +367,77 @@
|
||||
"terminate_process(reranker_process)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## v1/score (decoder-only scoring)\n",
|
||||
"\n",
|
||||
"Compute token probabilities for specified tokens given a query and items. This is useful for classification tasks, scoring responses, or computing log-probabilities.\n",
|
||||
"\n",
|
||||
"Parameters:\n",
|
||||
"- `query`: Query text\n",
|
||||
"- `items`: Item text(s) to score\n",
|
||||
"- `label_token_ids`: Token IDs to compute probabilities for\n",
|
||||
"- `apply_softmax`: Whether to apply softmax to get normalized probabilities (default: False)\n",
|
||||
"- `item_first`: Whether items come first in concatenation order (default: False)\n",
|
||||
"- `model`: Model name\n",
|
||||
"\n",
|
||||
"The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"score_process, port = launch_server_cmd(\n",
|
||||
" \"\"\"\n",
|
||||
"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n",
|
||||
" --host 0.0.0.0 --log-level warning\n",
|
||||
"\"\"\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"wait_for_server(f\"http://localhost:{port}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Score the probability of different completions given a query\n",
|
||||
"query = \"The capital of France is\"\n",
|
||||
"items = [\"Paris\", \"London\", \"Berlin\"]\n",
|
||||
"\n",
|
||||
"url = f\"http://localhost:{port}/v1/score\"\n",
|
||||
"data = {\n",
|
||||
" \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n",
|
||||
" \"query\": query,\n",
|
||||
" \"items\": items,\n",
|
||||
" \"label_token_ids\": [9454, 2753], # e.g. \"Yes\" and \"No\" token ids\n",
|
||||
" \"apply_softmax\": True, # Normalize probabilities to sum to 1\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"response = requests.post(url, json=data)\n",
|
||||
"response_json = response.json()\n",
|
||||
"\n",
|
||||
"# Display scores for each item\n",
|
||||
"for item, scores in zip(items, response_json[\"scores\"]):\n",
|
||||
" print_highlight(f\"Item '{item}': probabilities = {[f'{s:.4f}' for s in scores]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"terminate_process(score_process)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
||||
Reference in New Issue
Block a user