21 KiB
21 KiB
In [ ]:
from sglang.test.doc_patch import launch_server_cmd
from sglang.utils import wait_for_server, print_highlight, terminate_process
server_process, port = launch_server_cmd(
"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning"
)
wait_for_server(f"http://localhost:{port}", process=server_process)
print(f"Server started on http://localhost:{port}")In [ ]:
import openai
client = openai.Client(base_url=f"http://127.0.0.1:{port}/v1", api_key="None")
response = client.chat.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
messages=[
{"role": "user", "content": "List 3 countries and their capitals."},
],
temperature=0,
max_tokens=64,
)
print_highlight(f"Response: {response}")In [ ]:
# Example with logit_bias parameter
# Note: You need to get the actual token IDs from your tokenizer
# For demonstration, we'll use some example token IDs
response = client.chat.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
messages=[
{"role": "user", "content": "Complete this sentence: The weather today is"}
],
temperature=0.7,
max_tokens=20,
logit_bias={
"12345": 50, # Increase likelihood of token ID 12345
"67890": -50, # Decrease likelihood of token ID 67890
"11111": 25, # Slightly increase likelihood of token ID 11111
},
)
print_highlight(f"Response with logit bias: {response.choices[0].message.content}")In [ ]:
response = client.chat.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
messages=[
{
"role": "system",
"content": "You are a knowledgeable historian who provides concise responses.",
},
{"role": "user", "content": "Tell me about ancient Rome"},
{
"role": "assistant",
"content": "Ancient Rome was a civilization centered in Italy.",
},
{"role": "user", "content": "What were their major achievements?"},
],
temperature=0.3, # Lower temperature for more focused responses
max_tokens=128, # Reasonable length for a concise response
top_p=0.95, # Slightly higher for better fluency
presence_penalty=0.2, # Mild penalty to avoid repetition
frequency_penalty=0.2, # Mild penalty for more natural language
n=1, # Single response is usually more stable
seed=42, # Keep for reproducibility
)
print_highlight(response.choices[0].message.content)In [ ]:
stream = client.chat.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
messages=[{"role": "user", "content": "Say this is a test"}],
stream=True,
)
for chunk in stream:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")In [ ]:
# Example with logit_bias parameter for completions API
# Note: You need to get the actual token IDs from your tokenizer
# For demonstration, we'll use some example token IDs
response = client.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
prompt="The best programming language for AI is",
temperature=0.7,
max_tokens=20,
logit_bias={
"12345": 75, # Strongly favor token ID 12345
"67890": -100, # Completely avoid token ID 67890
"11111": -25, # Slightly discourage token ID 11111
},
)
print_highlight(f"Response with logit bias: {response.choices[0].text}")In [ ]:
response = client.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
prompt="List 3 countries and their capitals.",
temperature=0,
max_tokens=64,
n=1,
stop=None,
)
print_highlight(f"Response: {response}")In [ ]:
response = client.completions.create(
model="qwen/qwen2.5-0.5b-instruct",
prompt="Write a short story about a space explorer.",
temperature=0.7, # Moderate temperature for creative writing
max_tokens=150, # Longer response for a story
top_p=0.9, # Balanced diversity in word choice
stop=["\n\n", "THE END"], # Multiple stop sequences
presence_penalty=0.3, # Encourage novel elements
frequency_penalty=0.3, # Reduce repetitive phrases
n=1, # Generate one completion
seed=123, # For reproducible results
)
print_highlight(f"Response: {response}")In [ ]:
terminate_process(server_process)