diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9ef2c4d49..80569a220 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,9 +26,9 @@ jobs: run: SKIP=no-commit-to-branch pre-commit run --all-files --show-diff-on-failure - name: Run sgl-kernel clang-format checks - uses: DoozyX/clang-format-lint-action@v0.18.1 + uses: DoozyX/clang-format-lint-action@v0.20 with: source: sgl-kernel extensions: h,c,cpp,hpp,cu,cuh,cc - clangFormatVersion: 18 + clangFormatVersion: 20 style: file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb70d9213..f088453f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ exclude: ^(python/sglang/multimodal_gen/csrc|python/sglang/jit_kernel/flash_atte repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-symlinks - id: destroyed-symlinks @@ -21,12 +21,12 @@ repos: - id: debug-statements - id: no-commit-to-branch - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.7 + rev: v0.15.1 hooks: - id: ruff args: @@ -43,7 +43,7 @@ repos: python/sglang/srt/grpc/.*_pb2_grpc\.pyi$| )$ - repo: https://github.com/psf/black - rev: 24.10.0 + rev: 26.1.0 hooks: - id: black-jupyter exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' @@ -53,13 +53,13 @@ repos: - id: codespell args: ['--config', '.codespellrc'] - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v18.1.8 + rev: v20.1.7 hooks: - id: clang-format types_or: [c++, cuda] args: [--style=file, --verbose] - repo: https://github.com/kynan/nbstripout - rev: 0.8.1 + rev: 0.9.0 hooks: - id: nbstripout args: diff --git a/3rdparty/amd/tuning/benchmark_moe_rocm.py b/3rdparty/amd/tuning/benchmark_moe_rocm.py index af596d218..131b25270 100644 --- a/3rdparty/amd/tuning/benchmark_moe_rocm.py +++ b/3rdparty/amd/tuning/benchmark_moe_rocm.py @@ -187,10 +187,8 @@ def run_grid(bs, model, method, tp_size, dtype: str): configs = union_of_list_of_dicts(prune_configs_1, prune_configs_2) - print( - f"{bs=} || {len(full_configs)=} | {len(prune_configs_1)=} | \ - {len(prune_configs_2)=} | {len(configs)=}" - ) + print(f"{bs=} || {len(full_configs)=} | {len(prune_configs_1)=} | \ + {len(prune_configs_2)=} | {len(configs)=}") best_config = None best_time_us = 1e20 diff --git a/benchmark/fla/benchmark_layernorm_gated.py b/benchmark/fla/benchmark_layernorm_gated.py index 82440582b..e678d8c31 100644 --- a/benchmark/fla/benchmark_layernorm_gated.py +++ b/benchmark/fla/benchmark_layernorm_gated.py @@ -7,7 +7,9 @@ import torch from sglang.srt.layers.attention.fla.layernorm_gated import ( _layer_norm_fwd as layer_norm_fwd, ) -from sglang.srt.layers.attention.fla.layernorm_gated import rms_norm_ref +from sglang.srt.layers.attention.fla.layernorm_gated import ( + rms_norm_ref, +) def benchmark_layer_norm_fwd( diff --git a/benchmark/tip_suggestion/bench_other.py b/benchmark/tip_suggestion/bench_other.py index 2630081bd..6e3d098fe 100644 --- a/benchmark/tip_suggestion/bench_other.py +++ b/benchmark/tip_suggestion/bench_other.py @@ -13,8 +13,7 @@ number = 5 def expand_tip(topic, tip, generate): - s = ( - """Please expand a tip for a topic into a detailed paragraph. + s = """Please expand a tip for a topic into a detailed paragraph. Topic: staying healthy Tip: Regular Exercise @@ -28,12 +27,7 @@ Topic: writing a blog post Tip: structure your content effectively Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. -Topic: """ - + topic - + "\nTip: " - + tip - + "\nParagraph:" - ) +Topic: """ + topic + "\nTip: " + tip + "\nParagraph:" return generate(s, max_tokens=128, stop=["\n\n"]) diff --git a/benchmark/tip_suggestion/bench_sglang.py b/benchmark/tip_suggestion/bench_sglang.py index 86c476f97..ef78dce69 100644 --- a/benchmark/tip_suggestion/bench_sglang.py +++ b/benchmark/tip_suggestion/bench_sglang.py @@ -14,8 +14,7 @@ number = 5 @sgl.function def expand_tip(s, topic, tip): - s += ( - """Please expand a tip for a topic into a detailed paragraph. + s += """Please expand a tip for a topic into a detailed paragraph. Topic: staying healthy Tip: Regular Exercise @@ -29,12 +28,7 @@ Topic: writing a blog post Tip: structure your content effectively Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. -Topic: """ - + topic - + "\nTip: " - + tip - + "\nParagraph:" - ) +Topic: """ + topic + "\nTip: " + tip + "\nParagraph:" s += sgl.gen("paragraph", max_tokens=128, stop=["\n\n"], temperature=0) diff --git a/benchmark/tip_suggestion/lmql_funcs.py b/benchmark/tip_suggestion/lmql_funcs.py index 7790bbe95..1d4c97e38 100644 --- a/benchmark/tip_suggestion/lmql_funcs.py +++ b/benchmark/tip_suggestion/lmql_funcs.py @@ -2,8 +2,7 @@ number = 5 async def expand_tip_async(topic, tip, generate): - s = ( - """Please expand a tip for a topic into a detailed paragraph. + s = """Please expand a tip for a topic into a detailed paragraph. Topic: staying healthy Tip: Regular Exercise @@ -17,12 +16,7 @@ Topic: writing a blog post Tip: structure your content effectively Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. -Topic: """ - + topic - + "\nTip: " - + tip - + "\nParagraph:" - ) +Topic: """ + topic + "\nTip: " + tip + "\nParagraph:" return await generate(s, max_tokens=128, stop="\n\n") diff --git a/docs/advanced_features/lora.ipynb b/docs/advanced_features/lora.ipynb index 7be400b89..42380312e 100644 --- a/docs/advanced_features/lora.ipynb +++ b/docs/advanced_features/lora.ipynb @@ -151,16 +151,14 @@ "metadata": {}, "outputs": [], "source": [ - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \\\n", " --max-loras-per-batch 2 \\\n", " --log-level warning \\\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] @@ -227,8 +225,7 @@ "\n", "# The `--target-lora-modules` param below is technically not needed, as the server will infer it from lora0 which already has all the target modules specified.\n", "# We are adding it here just to demonstrate usage.\n", - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --cuda-graph-max-bs 2 \\\n", @@ -236,8 +233,7 @@ " --max-lora-rank 256\n", " --lora-target-modules all\n", " --log-level warning\n", - " \"\"\"\n", - ")\n", + " \"\"\")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", "wait_for_server(url, process=server_process)" @@ -435,8 +431,7 @@ "metadata": {}, "outputs": [], "source": [ - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --cuda-graph-max-bs 8 \\\n", @@ -448,8 +443,7 @@ " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", " --log-level warning\n", - " \"\"\"\n", - ")\n", + " \"\"\")\n", "\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", @@ -548,16 +542,14 @@ "metadata": {}, "outputs": [], "source": [ - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server \\\n", " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", " --lora-backend csgmv \\\n", " --max-loras-per-batch 16 \\\n", " --lora-paths lora1=path/to/lora1 lora2=path/to/lora2\n", - " \"\"\"\n", - ")" + " \"\"\")" ] }, { @@ -594,8 +586,7 @@ "lora2 = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\"\n", "\n", "\n", - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server \\\n", " --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", @@ -606,8 +597,7 @@ " --max-lora-rank 256 \\\n", " --max-loras-per-batch 2 \\\n", " --max-loaded-loras 4\n", - " \"\"\"\n", - ")\n", + " \"\"\")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", "wait_for_server(url, process=server_process)" diff --git a/docs/advanced_features/structured_outputs.ipynb b/docs/advanced_features/structured_outputs.ipynb index 6106d40d6..ec603e4e6 100644 --- a/docs/advanced_features/structured_outputs.ipynb +++ b/docs/advanced_features/structured_outputs.ipynb @@ -740,7 +740,6 @@ "import json\n", "from pydantic import BaseModel, Field\n", "\n", - "\n", "prompts = [\n", " \"Give me the information of the capital of China in the JSON format.\",\n", " \"Give me the information of the capital of France in the JSON format.\",\n", diff --git a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb index b12dbdfea..cfc07fd01 100644 --- a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb +++ b/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb @@ -642,7 +642,6 @@ "import json\n", "from pydantic import BaseModel, Field\n", "\n", - "\n", "prompts = [\n", " \"Give me the information of the capital of China in the JSON format.\",\n", " \"Give me the information of the capital of France in the JSON format.\",\n", diff --git a/docs/advanced_features/vlm_query.ipynb b/docs/advanced_features/vlm_query.ipynb index 45dd9a1ef..13491d1b9 100644 --- a/docs/advanced_features/vlm_query.ipynb +++ b/docs/advanced_features/vlm_query.ipynb @@ -117,7 +117,6 @@ "source": [ "from sglang import Engine\n", "\n", - "\n", "llm = Engine(model_path=model_path, chat_template=chat_template, log_level=\"warning\")" ] }, diff --git a/docs/basic_usage/native_api.ipynb b/docs/basic_usage/native_api.ipynb index 7a914fc3e..05f4f3688 100644 --- a/docs/basic_usage/native_api.ipynb +++ b/docs/basic_usage/native_api.ipynb @@ -275,12 +275,10 @@ "metadata": {}, "outputs": [], "source": [ - "embedding_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "embedding_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] @@ -324,12 +322,10 @@ "metadata": {}, "outputs": [], "source": [ - "reranker_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "reranker_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \\\n", " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)" ] @@ -392,12 +388,10 @@ "metadata": {}, "outputs": [], "source": [ - "score_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "score_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", " --host 0.0.0.0 --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=score_process)" ] @@ -456,11 +450,9 @@ "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", "# This will be updated in the future.\n", "\n", - "reward_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "reward_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=reward_process)" ] @@ -575,11 +567,9 @@ "metadata": {}, "outputs": [], "source": [ - "tokenizer_free_server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "tokenizer_free_server_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)" ] diff --git a/docs/basic_usage/openai_api_embeddings.ipynb b/docs/basic_usage/openai_api_embeddings.ipynb index 693931824..a6c90c06b 100644 --- a/docs/basic_usage/openai_api_embeddings.ipynb +++ b/docs/basic_usage/openai_api_embeddings.ipynb @@ -30,12 +30,10 @@ "from sglang.test.doc_patch import launch_server_cmd\n", "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", "\n", - "embedding_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "embedding_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", " --host 0.0.0.0 --is-embedding --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] diff --git a/docs/basic_usage/openai_api_vision.ipynb b/docs/basic_usage/openai_api_vision.ipynb index a6bb88f0c..45198bb3c 100644 --- a/docs/basic_usage/openai_api_vision.ipynb +++ b/docs/basic_usage/openai_api_vision.ipynb @@ -33,11 +33,9 @@ "from sglang.test.doc_patch import launch_server_cmd\n", "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", "\n", - "vision_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "vision_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=vision_process)" ] diff --git a/docs/basic_usage/send_request.ipynb b/docs/basic_usage/send_request.ipynb index fd5f87a27..968a23b8d 100644 --- a/docs/basic_usage/send_request.ipynb +++ b/docs/basic_usage/send_request.ipynb @@ -31,12 +31,10 @@ "# This is equivalent to running the following command in your terminal\n", "# python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0\n", "\n", - "server_process, port = launch_server_cmd(\n", - " \"\"\"\n", + "server_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", " --host 0.0.0.0 --log-level warning\n", - "\"\"\"\n", - ")\n", + "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] diff --git a/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb index f309142ad..eef47c15e 100644 --- a/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb +++ b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb @@ -107,7 +107,6 @@ "from sglang.lang.interpreter import ProgramState\n", "from parea import Parea, trace\n", "\n", - "\n", "load_dotenv()\n", "\n", "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", @@ -237,7 +236,6 @@ " percent_target_supported_by_context_factory,\n", ")\n", "\n", - "\n", "context_relevancy_eval = context_query_relevancy_factory()\n", "percent_target_supported_by_context = percent_target_supported_by_context_factory()\n", "\n", @@ -263,7 +261,6 @@ "from parea.evals.general import answer_matches_target_llm_grader_factory\n", "from parea.evals.rag import answer_context_faithfulness_statement_level_factory\n", "\n", - "\n", "answer_context_faithfulness = answer_context_faithfulness_statement_level_factory()\n", "answer_matches_target_llm_grader = answer_matches_target_llm_grader_factory()\n", "\n", diff --git a/examples/runtime/engine/save_remote_state.py b/examples/runtime/engine/save_remote_state.py index a428195ca..a5019d086 100644 --- a/examples/runtime/engine/save_remote_state.py +++ b/examples/runtime/engine/save_remote_state.py @@ -18,6 +18,7 @@ llm = Engine( tensor_parallel_size=8, ) """ + import dataclasses from argparse import ArgumentParser from pathlib import Path diff --git a/examples/runtime/engine/save_sharded_state.py b/examples/runtime/engine/save_sharded_state.py index 80ad5321f..69665e35e 100644 --- a/examples/runtime/engine/save_sharded_state.py +++ b/examples/runtime/engine/save_sharded_state.py @@ -21,6 +21,7 @@ llm = Engine( tensor_parallel_size=8, ) """ + import dataclasses import os import shutil diff --git a/python/sglang/jit_kernel/include/sgl_kernel/type.cuh b/python/sglang/jit_kernel/include/sgl_kernel/type.cuh index ff31c5ff3..4b6723855 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/type.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/type.cuh @@ -36,27 +36,28 @@ struct dtype_trait {}; } \ static_assert(true) -SGL_REGISTER_DTYPE_TRAIT(fp32_t, fp32x2_t, SGL_REGISTER_TYPE_END; // - SGL_REGISTER_FROM_FUNCTION(fp16_t, __half2float); - SGL_REGISTER_FROM_FUNCTION(bf16_t, __bfloat162float); - SGL_REGISTER_UNARY_FUNCTION(abs, fabsf); - SGL_REGISTER_UNARY_FUNCTION(sqrt, sqrtf); - SGL_REGISTER_UNARY_FUNCTION(rsqrt, rsqrtf); - SGL_REGISTER_BINARY_FUNCTION(max, fmaxf); - SGL_REGISTER_BINARY_FUNCTION(min, fminf);); +SGL_REGISTER_DTYPE_TRAIT( + fp32_t, fp32x2_t, SGL_REGISTER_TYPE_END; // + SGL_REGISTER_FROM_FUNCTION(fp16_t, __half2float); + SGL_REGISTER_FROM_FUNCTION(bf16_t, __bfloat162float); + SGL_REGISTER_UNARY_FUNCTION(abs, fabsf); + SGL_REGISTER_UNARY_FUNCTION(sqrt, sqrtf); + SGL_REGISTER_UNARY_FUNCTION(rsqrt, rsqrtf); + SGL_REGISTER_BINARY_FUNCTION(max, fmaxf); + SGL_REGISTER_BINARY_FUNCTION(min, fminf);); SGL_REGISTER_DTYPE_TRAIT(fp16_t, fp16x2_t); SGL_REGISTER_DTYPE_TRAIT(bf16_t, bf16x2_t); /// TODO: Add ROCM implementation -SGL_REGISTER_DTYPE_TRAIT(fp32x2_t, fp32x4_t, SGL_REGISTER_TYPE_END; - SGL_REGISTER_FROM_FUNCTION(fp16x2_t, __half22float2); - SGL_REGISTER_FROM_FUNCTION(bf16x2_t, __bfloat1622float2);); +SGL_REGISTER_DTYPE_TRAIT( + fp32x2_t, fp32x4_t, SGL_REGISTER_TYPE_END; SGL_REGISTER_FROM_FUNCTION(fp16x2_t, __half22float2); + SGL_REGISTER_FROM_FUNCTION(bf16x2_t, __bfloat1622float2);); -SGL_REGISTER_DTYPE_TRAIT(fp16x2_t, void, SGL_REGISTER_TYPE_END; - SGL_REGISTER_FROM_FUNCTION(fp32x2_t, __float22half2_rn);); +SGL_REGISTER_DTYPE_TRAIT( + fp16x2_t, void, SGL_REGISTER_TYPE_END; SGL_REGISTER_FROM_FUNCTION(fp32x2_t, __float22half2_rn);); -SGL_REGISTER_DTYPE_TRAIT(bf16x2_t, void, SGL_REGISTER_TYPE_END; - SGL_REGISTER_FROM_FUNCTION(fp32x2_t, __float22bfloat162_rn);); +SGL_REGISTER_DTYPE_TRAIT( + bf16x2_t, void, SGL_REGISTER_TYPE_END; SGL_REGISTER_FROM_FUNCTION(fp32x2_t, __float22bfloat162_rn);); #undef SGL_REGISTER_DTYPE_TRAIT #undef SGL_REGISTER_FROM_FUNCTION diff --git a/python/sglang/multimodal_gen/apps/webui/main.py b/python/sglang/multimodal_gen/apps/webui/main.py index f16fb6cf8..56816309b 100644 --- a/python/sglang/multimodal_gen/apps/webui/main.py +++ b/python/sglang/multimodal_gen/apps/webui/main.py @@ -243,12 +243,10 @@ def run_sgl_diffusion_webui(server_args: ServerArgs): # print banner delimiter = "=" * 80 url = local_url or f"http://localhost:{server_args.webui_port}" - print( - f""" + print(f""" {delimiter} \033[1mSGLang Diffusion WebUI available at:\033[0m \033[1;4;92m{url}\033[0m {delimiter} -""" - ) +""") demo.block_thread() diff --git a/python/sglang/multimodal_gen/configs/models/encoders/qwen3.py b/python/sglang/multimodal_gen/configs/models/encoders/qwen3.py index 3b56e690d..ed48da96a 100644 --- a/python/sglang/multimodal_gen/configs/models/encoders/qwen3.py +++ b/python/sglang/multimodal_gen/configs/models/encoders/qwen3.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Qwen3 text encoder configuration for SGLang diffusion models.""" + from dataclasses import dataclass, field from sglang.multimodal_gen.configs.models.encoders.base import ( diff --git a/python/sglang/multimodal_gen/runtime/cache/__init__.py b/python/sglang/multimodal_gen/runtime/cache/__init__.py index f01f41cc0..62f0f8457 100644 --- a/python/sglang/multimodal_gen/runtime/cache/__init__.py +++ b/python/sglang/multimodal_gen/runtime/cache/__init__.py @@ -9,6 +9,7 @@ diffusion transformer (DiT) inference: - cache-dit integration: Block-level caching with DBCache and TaylorSeer """ + from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( CacheDitConfig, enable_cache_on_dual_transformer, diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py index 7bc7d8b4e..18edea976 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py +++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py @@ -29,6 +29,7 @@ The typical workflow is: If you only need to use the distributed environment without model parallelism, you can skip the model parallel initialization and destruction steps. """ + import contextlib import datetime import os @@ -71,7 +72,7 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) def _split_tensor_dict( - tensor_dict: dict[str, torch.Tensor | Any] + tensor_dict: dict[str, torch.Tensor | Any], ) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]: """Split the tensor dictionary into two parts: 1. A list of (key, value) pairs. If the value is a tensor, it is replaced diff --git a/python/sglang/multimodal_gen/runtime/layers/activation.py b/python/sglang/multimodal_gen/runtime/layers/activation.py index 43b325950..90e9289fe 100644 --- a/python/sglang/multimodal_gen/runtime/layers/activation.py +++ b/python/sglang/multimodal_gen/runtime/layers/activation.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/activation.py """Custom activation functions.""" + import math from typing import Any diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index 4d550787c..f92fce78b 100644 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/layernorm.py """Custom normalization layers.""" + from typing import Optional, Tuple, Union import torch diff --git a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py index 00acf5123..2a8a65993 100644 --- a/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py +++ b/python/sglang/multimodal_gen/runtime/layers/rotary_embedding.py @@ -25,6 +25,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Rotary Positional Embeddings.""" + import functools from collections import OrderedDict from typing import Any, Optional, Tuple diff --git a/python/sglang/multimodal_gen/runtime/layers/utils.py b/python/sglang/multimodal_gen/runtime/layers/utils.py index c41515288..2777e9209 100644 --- a/python/sglang/multimodal_gen/runtime/layers/utils.py +++ b/python/sglang/multimodal_gen/runtime/layers/utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/utils.py """Utility methods for model layers.""" + import inspect from typing import Any, Callable, List, Optional diff --git a/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py b/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py index 535bf3311..c8eff7d6f 100644 --- a/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py +++ b/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py @@ -12,7 +12,10 @@ from diffusers.models.embeddings import ( from diffusers.models.embeddings import ( CombinedTimestepTextProjEmbeddings as _CombinedTimestepTextProjEmbeddings, ) -from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbedding +from diffusers.models.embeddings import ( + PixArtAlphaTextProjection, + TimestepEmbedding, +) from diffusers.models.embeddings import Timesteps as _Timesteps from diffusers.models.embeddings import ( get_timestep_embedding as timestep_embedding_diffusers, diff --git a/python/sglang/multimodal_gen/runtime/loader/utils.py b/python/sglang/multimodal_gen/runtime/loader/utils.py index 4be14ef41..3f0310011 100644 --- a/python/sglang/multimodal_gen/runtime/loader/utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/utils.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Utilities for selecting and loading models.""" + import contextlib import glob import os @@ -30,7 +31,7 @@ def set_default_torch_dtype(dtype: torch.dtype): def get_param_names_mapping( - mapping_dict: dict[str, str] + mapping_dict: dict[str, str], ) -> Callable[[str], tuple[str, Any, Any]]: """ Creates a mapping function that transforms parameter names using regex patterns. diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py index 89d22b31e..e74c40c17 100644 --- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/model_loader/weight_utils.py """Utilities for downloading and initializing model weights.""" + import hashlib import json import os diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/clip.py b/python/sglang/multimodal_gen/runtime/models/encoders/clip.py index 4b85a1260..83fdefd8c 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/clip.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/clip.py @@ -5,6 +5,7 @@ # Adapted from transformers: https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py """Minimal implementation of CLIPVisionModel intended to be only used within a vision language model.""" + from collections.abc import Iterable from typing import Optional diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py index ea208f124..a9d209231 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/llama.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/llama.py @@ -25,6 +25,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only LLaMA model compatible with HuggingFace weights.""" + from collections.abc import Iterable from typing import Any diff --git a/python/sglang/multimodal_gen/runtime/models/utils.py b/python/sglang/multimodal_gen/runtime/models/utils.py index 331940bc6..0199bbb3d 100644 --- a/python/sglang/multimodal_gen/runtime/models/utils.py +++ b/python/sglang/multimodal_gen/runtime/models/utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/utils.py """Utils for model executor.""" + from typing import Any import torch diff --git a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py index 0be68fc4d..9a829ab14 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/hunyuan_pipeline.py @@ -8,7 +8,6 @@ This module contains an implementation of the Hunyuan video diffusion pipeline using the modular pipeline architecture. """ - from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( ComposedPipelineBase, ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py index 9af35e9be..5c1d670e6 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/sync_executor.py @@ -4,6 +4,7 @@ """ Synchronous pipeline executor implementation. """ + from typing import List from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/comfyui_latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/comfyui_latent_preparation.py index ece5dcb88..0b4e53a92 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/comfyui_latent_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/comfyui_latent_preparation.py @@ -5,6 +5,7 @@ This stage extends LatentPreparationStage to handle device mismatch issues that occur when tensors are pickled and unpickled via broadcast_pyobj in multi-GPU scenarios. """ + import dataclasses import torch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py index f4b85c260..b766a9c66 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/input_validation.py @@ -4,6 +4,7 @@ """ Input validation stage for diffusion pipelines. """ + import numpy as np import torch import torchvision.transforms.functional as TF diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py index 1c0675905..dbae650c7 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py @@ -4,6 +4,7 @@ """ Latent preparation stage for diffusion pipelines. """ + from diffusers.utils.torch_utils import randn_tensor from sglang.multimodal_gen.runtime.distributed import get_local_torch_device diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py index 02eb2ffa4..d36c76dc0 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py +++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -6,6 +6,7 @@ This file is a platform abstraction for ROCm GPUs, adjusted to match the structure and interface of `cuda.py`. """ + from functools import lru_cache from typing import Any diff --git a/python/sglang/multimodal_gen/runtime/server_args.py b/python/sglang/multimodal_gen/runtime/server_args.py index edc1da2d9..9d039f58b 100644 --- a/python/sglang/multimodal_gen/runtime/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Inspired by SGLang: https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py """The arguments of sglang-diffusion Inference.""" + import argparse import dataclasses import inspect diff --git a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py index b73fafabe..2b94bfe8b 100644 --- a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/logger.py """Logging configuration for sglang.multimodal_gen.""" + import argparse import contextlib import datetime diff --git a/python/sglang/multimodal_gen/test/cli/test_generate_common.py b/python/sglang/multimodal_gen/test/cli/test_generate_common.py index 5303bed1e..4bcbf8ab1 100644 --- a/python/sglang/multimodal_gen/test/cli/test_generate_common.py +++ b/python/sglang/multimodal_gen/test/cli/test_generate_common.py @@ -1,8 +1,9 @@ # Copied and adapted from: https://github.com/hao-ai-lab/FastVideo """ - Common generate cli test, one test for image and video each +Common generate cli test, one test for image and video each """ + import dataclasses import os import shlex diff --git a/python/sglang/multimodal_gen/test/slack_utils.py b/python/sglang/multimodal_gen/test/slack_utils.py index 4f4c2a6b7..34ba78371 100644 --- a/python/sglang/multimodal_gen/test/slack_utils.py +++ b/python/sglang/multimodal_gen/test/slack_utils.py @@ -1,5 +1,5 @@ """ - This file upload the media generated in diffusion-nightly-test to a slack channel of SGLang +This file upload the media generated in diffusion-nightly-test to a slack channel of SGLang """ import logging diff --git a/python/sglang/multimodal_gen/third_party/pynvml.py b/python/sglang/multimodal_gen/third_party/pynvml.py index 546dc8b8b..24d0a714c 100644 --- a/python/sglang/multimodal_gen/third_party/pynvml.py +++ b/python/sglang/multimodal_gen/third_party/pynvml.py @@ -4855,7 +4855,7 @@ def nvmlDeviceGetFieldValues(handle, fieldIds): for i, fieldId in enumerate(fieldIds): try: - (values[i].fieldId, values[i].scopeId) = fieldId + values[i].fieldId, values[i].scopeId = fieldId except TypeError: values[i].fieldId = fieldId @@ -4871,7 +4871,7 @@ def nvmlDeviceClearFieldValues(handle, fieldIds): for i, fieldId in enumerate(fieldIds): try: - (values[i].fieldId, values[i].scopeId) = fieldId + values[i].fieldId, values[i].scopeId = fieldId except TypeError: values[i].fieldId = fieldId diff --git a/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py b/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py index dd8805e65..6f11c7872 100644 --- a/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py +++ b/python/sglang/srt/checkpoint_engine/checkpoint_engine_worker.py @@ -15,6 +15,7 @@ Checkpoint-engine integration for SGLang. This module provides weight update functionality via IPC for checkpoint-engine compatibility. """ + import logging from typing import Callable, Dict, Optional diff --git a/python/sglang/srt/compilation/weak_ref_tensor.py b/python/sglang/srt/compilation/weak_ref_tensor.py index 30f59d46f..3564849c7 100644 --- a/python/sglang/srt/compilation/weak_ref_tensor.py +++ b/python/sglang/srt/compilation/weak_ref_tensor.py @@ -13,7 +13,7 @@ else: def weak_ref_tensors( - tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]] + tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]], ) -> Union[torch.Tensor, list[Any], tuple[Any], Any]: """ Convenience function to create weak references to tensors, diff --git a/python/sglang/srt/configs/exaone.py b/python/sglang/srt/configs/exaone.py index 7b0a2d290..f5d91c45c 100644 --- a/python/sglang/srt/configs/exaone.py +++ b/python/sglang/srt/configs/exaone.py @@ -14,7 +14,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" EXAONE model configuration """ +"""EXAONE model configuration""" + from typing import Any, Dict from transformers.configuration_utils import PretrainedConfig diff --git a/python/sglang/srt/configs/falcon_h1.py b/python/sglang/srt/configs/falcon_h1.py index eae690071..3aa038fb8 100644 --- a/python/sglang/srt/configs/falcon_h1.py +++ b/python/sglang/srt/configs/falcon_h1.py @@ -14,7 +14,6 @@ # limitations under the License. """Falcon-H1 model configuration""" - from transformers.configuration_utils import PretrainedConfig from transformers.utils import logging diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 93b6395fd..25c82f8a7 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -970,7 +970,7 @@ class MooncakeKVManager(CommonKVManager): self._handle_aux_data(msg) continue - (bootstrap_room, status, prefill_rank) = msg + bootstrap_room, status, prefill_rank = msg status = int(status.decode("ascii")) bootstrap_room = int(bootstrap_room.decode("ascii")) prefill_rank = int(prefill_rank.decode("ascii")) diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index d7d12c744..2c4e99c38 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -21,6 +21,7 @@ If you only need to use the distributed environment without model/pipeline parallelism, you can skip the model parallel initialization and destruction steps. """ + import contextlib import gc import logging @@ -79,7 +80,7 @@ class P2PWork: def _split_tensor_dict( - tensor_dict: Dict[str, Union[torch.Tensor, Any]] + tensor_dict: Dict[str, Union[torch.Tensor, Any]], ) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]: """Split the tensor dictionary into two parts: 1. A list of (key, value) pairs. If the value is a tensor, it is replaced diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index d2c46ed06..edd4e52cf 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from vLLM's OpenAIServingResponses """Handler for /v1/responses requests""" + from __future__ import annotations import asyncio diff --git a/python/sglang/srt/function_call/base_format_detector.py b/python/sglang/srt/function_call/base_format_detector.py index d4532761c..8022dbe07 100644 --- a/python/sglang/srt/function_call/base_format_detector.py +++ b/python/sglang/srt/function_call/base_format_detector.py @@ -186,7 +186,7 @@ class BaseFormatDetector(ABC): if start_idx >= len(current_text): return StreamingParseResult() - (obj, end_idx) = _partial_json_loads(current_text[start_idx:], flags) + obj, end_idx = _partial_json_loads(current_text[start_idx:], flags) is_current_complete = _is_complete_json( current_text[start_idx : start_idx + end_idx] diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py index bad71cf56..4ffc5fdd2 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -""" Run the model with npu graph and torch.compile """ +"""Run the model with npu graph and torch.compile""" from __future__ import annotations diff --git a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py index 775e03bb2..e522fbe4a 100644 --- a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py +++ b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Attention layer with Dual chunk flash attention and sparse attention. -""" +"""Attention layer with Dual chunk flash attention and sparse attention.""" + import functools import logging import math diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index c88c79652..360d164e5 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -670,9 +670,9 @@ class KimiLinearAttnBackend(MambaAttnBackendBase): **kwargs, ): assert isinstance(mixed_qkv, Tuple) - (q_proj_states, k_proj_states, v_proj_states) = mixed_qkv - (q_conv_weights, k_conv_weights, v_conv_weights) = layer.conv_weights - (q_conv_bias, k_conv_bias, v_conv_bias) = layer.bias + q_proj_states, k_proj_states, v_proj_states = mixed_qkv + q_conv_weights, k_conv_weights, v_conv_weights = layer.conv_weights + q_conv_bias, k_conv_bias, v_conv_bias = layer.bias layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer.layer_id) q_conv_state, k_conv_state, v_conv_state = layer_cache.conv @@ -744,9 +744,9 @@ class KimiLinearAttnBackend(MambaAttnBackendBase): ) assert isinstance(mixed_qkv, Tuple) - (q_proj_states, k_proj_states, v_proj_states) = mixed_qkv - (q_conv_weights, k_conv_weights, v_conv_weights) = layer.conv_weights - (q_conv_bias, k_conv_bias, v_conv_bias) = layer.bias + q_proj_states, k_proj_states, v_proj_states = mixed_qkv + q_conv_weights, k_conv_weights, v_conv_weights = layer.conv_weights + q_conv_bias, k_conv_bias, v_conv_bias = layer.bias query_start_loc = self.forward_metadata.query_start_loc cache_indices = self.forward_metadata.mamba_cache_indices diff --git a/python/sglang/srt/layers/attention/linear/seg_la.py b/python/sglang/srt/layers/attention/linear/seg_la.py index 4519ff1a3..9b4e68d8d 100644 --- a/python/sglang/srt/layers/attention/linear/seg_la.py +++ b/python/sglang/srt/layers/attention/linear/seg_la.py @@ -2,6 +2,7 @@ """ Copyright (c) Ant Financial Service Group and its affiliates. """ + # Copied from https://code.alipay.com/pia/PainlessInferenceAcceleration/blob/v0.0.6/flood/flood/ops/seg_la.py from dataclasses import dataclass diff --git a/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py b/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py index 2d8aa4678..ff446831b 100644 --- a/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py +++ b/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py @@ -38,7 +38,7 @@ def prefill_attention_wave( output_shape = (shape.total_seq_len, shape.num_query_heads, shape.head_size_kv) # Run the wave kernel. mfma_variant = (MMAType.F32_16x16x16_F16, MMAType.F32_16x16x16_F16) - (prefill, hyperparams) = get_prefill_attention_kernel( + prefill, hyperparams = get_prefill_attention_kernel( shape, mfma_variant, q.shape, diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 49c30a1d1..aff05bf42 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -376,7 +376,7 @@ class LogitsProcessor(nn.Module): logprobs_result = self.process_input_logprobs(input_logits, logits_metadata) else: - (logprobs_result, sampled_logits) = self.process_input_logprobs_by_chunk( + logprobs_result, sampled_logits = self.process_input_logprobs_by_chunk( pruned_states, sample_indices, input_logprob_indices, diff --git a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py index bed37dd78..6ce5a3ddc 100644 --- a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 """Cutlass W4A8 MoE kernel.""" + from typing import Optional import torch diff --git a/python/sglang/srt/layers/quantization/modelslim/modelslim_moe.py b/python/sglang/srt/layers/quantization/modelslim/modelslim_moe.py index 095d09f31..c1674249f 100644 --- a/python/sglang/srt/layers/quantization/modelslim/modelslim_moe.py +++ b/python/sglang/srt/layers/quantization/modelslim/modelslim_moe.py @@ -63,11 +63,9 @@ class ModelSlimMoEMethod(FusedMoEMethodBase): logger.info_once("Using ModelSlimW8A8Int8MoE") return ModelSlimW8A8Int8MoE(quant_config) else: - logger.warning( - f"Unsupported FusedMoe modelslim scheme: \ + logger.warning(f"Unsupported FusedMoe modelslim scheme: \ {quant_config.quant_description.get(prefix_in_quant_config.strip())} \ - in layer: {prefix}" - ) + in layer: {prefix}") return None diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py index dd6f8ad29..449bc8670 100644 --- a/python/sglang/srt/layers/radix_attention.py +++ b/python/sglang/srt/layers/radix_attention.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """Radix attention.""" + from __future__ import annotations from enum import Enum diff --git a/python/sglang/srt/layers/radix_linear_attention.py b/python/sglang/srt/layers/radix_linear_attention.py index 7955cb0a0..571d292c0 100644 --- a/python/sglang/srt/layers/radix_linear_attention.py +++ b/python/sglang/srt/layers/radix_linear_attention.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """Radix linear attention.""" + from __future__ import annotations from typing import TYPE_CHECKING, Optional, Tuple, Union diff --git a/python/sglang/srt/layers/rotary_embedding.py b/python/sglang/srt/layers/rotary_embedding.py index 52aee2404..4ff07a6b0 100644 --- a/python/sglang/srt/layers/rotary_embedding.py +++ b/python/sglang/srt/layers/rotary_embedding.py @@ -1,5 +1,6 @@ # Adapted from https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.6.6.post1/vllm/model_executor/layers/rotary_embedding.py """Rotary Positional Embeddings.""" + from __future__ import annotations import itertools @@ -2896,7 +2897,7 @@ class DualChunkRotaryEmbedding(MultiPlatformOp): self.local_size = local_size self.dtype = dtype self.device = torch.device(f"cuda:{torch.cuda.current_device()}") - (q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache) = ( + q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache = ( self._compute_cos_sin_cache() ) diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 9dcfc9f3d..86b009df4 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """A tensor parallel worker.""" + from __future__ import annotations import logging diff --git a/python/sglang/srt/mem_cache/mamba_radix_cache.py b/python/sglang/srt/mem_cache/mamba_radix_cache.py index 2a501b572..ebd5f6e82 100644 --- a/python/sglang/srt/mem_cache/mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/mamba_radix_cache.py @@ -639,7 +639,7 @@ class MambaRadixCache(BasePrefixCache): match_result = self.match_prefix( MatchPrefixParams(key=RadixKey(page_aligned_token_ids, req.extra_key)) ) - (new_indices, new_last_node) = ( + new_indices, new_last_node = ( match_result.device_indices, match_result.last_device_node, ) diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index ba969cda5..f3f7c0bb5 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -8,7 +8,9 @@ from typing import Optional import psutil import torch -from sglang.jit_kernel.hicache import can_use_hicache_jit_kernel +from sglang.jit_kernel.hicache import ( + can_use_hicache_jit_kernel, +) from sglang.jit_kernel.hicache import ( transfer_hicache_all_layer as jit_transfer_hicache_all_layer, ) diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index a4214dcc1..42b169728 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -523,7 +523,7 @@ class RadixCache(BasePrefixCache): # The prefix indices could be updated, reuse it match_result = self.match_prefix(MatchPrefixParams(key=radix_key)) - (new_indices, new_last_node) = ( + new_indices, new_last_node = ( match_result.device_indices, match_result.last_device_node, ) diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 8deadf251..1813a18ee 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -556,7 +556,7 @@ class SWARadixCache(BasePrefixCache): match_result = self.match_prefix( MatchPrefixParams(key=RadixKey(page_aligned_token_ids, req.extra_key)) ) - (new_indices, new_last_node) = ( + new_indices, new_last_node = ( match_result.device_indices, match_result.last_device_node, ) diff --git a/python/sglang/srt/metrics/collector.py b/python/sglang/srt/metrics/collector.py index 10f232d5a..ac6570803 100644 --- a/python/sglang/srt/metrics/collector.py +++ b/python/sglang/srt/metrics/collector.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """Utilities for Prometheus Metrics Collection.""" + import dataclasses import logging import os diff --git a/python/sglang/srt/metrics/utils.py b/python/sglang/srt/metrics/utils.py index 4dc498df7..71141341d 100644 --- a/python/sglang/srt/metrics/utils.py +++ b/python/sglang/srt/metrics/utils.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """Utilities for Prometheus Metrics.""" + import math from typing import List diff --git a/python/sglang/srt/model_loader/utils.py b/python/sglang/srt/model_loader/utils.py index 8f38e20ad..18739ed69 100644 --- a/python/sglang/srt/model_loader/utils.py +++ b/python/sglang/srt/model_loader/utils.py @@ -1,6 +1,7 @@ # Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/model_loader/utils.py """Utilities for selecting and loading models.""" + import concurrent.futures import contextlib import logging diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index dac219058..e2256b350 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -1,6 +1,7 @@ # Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/model_loader/weight_utils.py """Utilities for downloading and initializing model weights.""" + import collections import concurrent.futures import fnmatch @@ -140,12 +141,10 @@ def convert_bin_to_safetensor_file( sf_size = os.stat(sf_filename).st_size pt_size = os.stat(pt_filename).st_size if (sf_size - pt_size) / pt_size > 0.01: - raise RuntimeError( - f"""The file size different is more than 1%: + raise RuntimeError(f"""The file size different is more than 1%: - {sf_filename}: {sf_size} - {pt_filename}: {pt_size} - """ - ) + """) # check if the tensors are the same reloaded = safetensors.torch.load_file(sf_filename) diff --git a/python/sglang/srt/models/baichuan.py b/python/sglang/srt/models/baichuan.py index 84596ba1f..6d060b881 100644 --- a/python/sglang/srt/models/baichuan.py +++ b/python/sglang/srt/models/baichuan.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only BaiChuan model compatible with HuggingFace weights.""" + import math from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index a04e3d647..38eb17aba 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """SGLang BailingMoE model.""" + import logging from typing import Iterable, List, Optional, Tuple, Union diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index a365acdff..fcbae6bba 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """SGLang BailingMoENextN model.""" + import logging from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index a64239031..2fd07ea3d 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -13,6 +13,7 @@ # ============================================================================== """Inference-only DeepSeek NextN Speculative Decoding.""" + import logging from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/deepseek_ocr.py b/python/sglang/srt/models/deepseek_ocr.py index b3506606c..b143ddfa4 100644 --- a/python/sglang/srt/models/deepseek_ocr.py +++ b/python/sglang/srt/models/deepseek_ocr.py @@ -16,6 +16,7 @@ # Adapted from # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/llama.py#L1 """Inference-only Apertus model compatible with HuggingFace weights.""" + import copy import logging import math diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 064e91cf0..35fe54e28 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -15,6 +15,7 @@ # Adapted from: # https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py """Inference-only DeepseekV2 model.""" + from __future__ import annotations import logging diff --git a/python/sglang/srt/models/ernie4.py b/python/sglang/srt/models/ernie4.py index dffd8f09a..3a61d8fdc 100644 --- a/python/sglang/srt/models/ernie4.py +++ b/python/sglang/srt/models/ernie4.py @@ -12,7 +12,7 @@ # limitations under the License. # ============================================================================== -""" Inference-only Ernie4.5 model compatible with baidu/ERNIE-4.5-*-PT weights. """ +"""Inference-only Ernie4.5 model compatible with baidu/ERNIE-4.5-*-PT weights.""" from typing import Iterable, List, Optional, Tuple, Union diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index 3fe0fc6a7..24b37ce2b 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -12,7 +12,7 @@ # limitations under the License. # ============================================================================== -""" Inference-only Ernie4.5 VL model compatible with baidu/ERNIE-4.5-VL-*-PT weights. """ +"""Inference-only Ernie4.5 VL model compatible with baidu/ERNIE-4.5-VL-*-PT weights.""" import logging from itertools import islice diff --git a/python/sglang/srt/models/ernie45_vl.py b/python/sglang/srt/models/ernie45_vl.py index 9ce3e97ce..c371de793 100644 --- a/python/sglang/srt/models/ernie45_vl.py +++ b/python/sglang/srt/models/ernie45_vl.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only Ernie45-VL model compatible with HuggingFace weights.""" + import logging from functools import lru_cache, partial from typing import Iterable, List, Optional, Tuple, Type diff --git a/python/sglang/srt/models/ernie4_eagle.py b/python/sglang/srt/models/ernie4_eagle.py index bf62d515c..3b9a96fa9 100644 --- a/python/sglang/srt/models/ernie4_eagle.py +++ b/python/sglang/srt/models/ernie4_eagle.py @@ -12,7 +12,7 @@ # limitations under the License. # ============================================================================== -""" Ernie4.5 MTP model compatible with baidu/ERNIE-4.5-*-PT weights. """ +"""Ernie4.5 MTP model compatible with baidu/ERNIE-4.5-*-PT weights.""" from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/gpt2.py b/python/sglang/srt/models/gpt2.py index 1ec33406f..6dac103e2 100644 --- a/python/sglang/srt/models/gpt2.py +++ b/python/sglang/srt/models/gpt2.py @@ -17,6 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only GPT-2 model compatible with HuggingFace weights.""" + from typing import Iterable, Optional, Tuple, Type import torch diff --git a/python/sglang/srt/models/hunyuan.py b/python/sglang/srt/models/hunyuan.py index 300493a3f..0128d0095 100644 --- a/python/sglang/srt/models/hunyuan.py +++ b/python/sglang/srt/models/hunyuan.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only HunYuan model compatible with HuggingFace weights.""" + import re from typing import Any, Dict, Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 7094be5c5..cf0095551 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -18,6 +18,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """SGLang LLaDA2MoeModelLM model.""" + import logging from typing import Iterable, Optional, Tuple, Union diff --git a/python/sglang/srt/models/mllama.py b/python/sglang/srt/models/mllama.py index e0d5713d7..b93566926 100644 --- a/python/sglang/srt/models/mllama.py +++ b/python/sglang/srt/models/mllama.py @@ -1,6 +1,7 @@ # Adapted from: # https://github.com/vllm-project/vllm/blob/7193774b1ff8603ad5bf4598e5efba0d9a39b436/vllm/model_executor/models/mllama.py """PyTorch Mllama model.""" + import math from typing import Iterable, List, Optional, Tuple, Union diff --git a/python/sglang/srt/models/nemotron_nas.py b/python/sglang/srt/models/nemotron_nas.py index ebf49f95a..ac1ccd231 100644 --- a/python/sglang/srt/models/nemotron_nas.py +++ b/python/sglang/srt/models/nemotron_nas.py @@ -14,6 +14,7 @@ # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/nemotron_nas.py """Inference-only deci model compatible with HuggingFace weights.""" + from typing import Iterable, Optional, Tuple, Type, Union import torch diff --git a/python/sglang/srt/models/olmo.py b/python/sglang/srt/models/olmo.py index 0c1ecf857..091b08e8a 100644 --- a/python/sglang/srt/models/olmo.py +++ b/python/sglang/srt/models/olmo.py @@ -15,6 +15,7 @@ # Adapted from # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/olmo.py#L1 """Inference-only OLMo model compatible with HuggingFace weights.""" + from typing import Iterable, Optional, Tuple import torch diff --git a/python/sglang/srt/models/olmo2.py b/python/sglang/srt/models/olmo2.py index 8789a3477..2b1c6fa89 100644 --- a/python/sglang/srt/models/olmo2.py +++ b/python/sglang/srt/models/olmo2.py @@ -15,6 +15,7 @@ # Adapted from # https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/olmo2.py """Inference-only OLMo2 model compatible with HuggingFace weights.""" + from functools import partial from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/opt.py b/python/sglang/srt/models/opt.py index 0b2f0edb7..7db9250af 100644 --- a/python/sglang/srt/models/opt.py +++ b/python/sglang/srt/models/opt.py @@ -13,6 +13,7 @@ # ============================================================================== """Inference-only OPT model compatible with HuggingFace weights.""" + import logging from collections.abc import Iterable from typing import Optional, Union diff --git a/python/sglang/srt/models/orion.py b/python/sglang/srt/models/orion.py index cc444d394..2aee4b9aa 100644 --- a/python/sglang/srt/models/orion.py +++ b/python/sglang/srt/models/orion.py @@ -7,6 +7,7 @@ # LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/orion.py """Inference-only Orion-14B model compatible with HuggingFace weights.""" + from collections.abc import Iterable from typing import Any, Optional, Tuple diff --git a/python/sglang/srt/models/pixtral.py b/python/sglang/srt/models/pixtral.py index c59770f45..265801901 100644 --- a/python/sglang/srt/models/pixtral.py +++ b/python/sglang/srt/models/pixtral.py @@ -23,11 +23,15 @@ import torch import torch.nn as nn import torch.nn.functional as F from transformers import PixtralVisionConfig, PretrainedConfig -from transformers.models.pixtral.modeling_pixtral import PixtralRotaryEmbedding +from transformers.models.pixtral.modeling_pixtral import ( + PixtralRotaryEmbedding, +) from transformers.models.pixtral.modeling_pixtral import ( generate_block_attention_mask as _get_pixtral_attention_mask, ) -from transformers.models.pixtral.modeling_pixtral import position_ids_in_meshgrid +from transformers.models.pixtral.modeling_pixtral import ( + position_ids_in_meshgrid, +) from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 0760c4645..a3cfde4d6 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -15,6 +15,7 @@ # Adapted from llama2.py # Modify details for the adaptation of Qwen2 model. """Inference-only Qwen2 model compatible with HuggingFace weights.""" + import logging from typing import Any, Dict, Iterable, List, Optional, Tuple, Union diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index da64dc2de..8b6b2b17d 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -22,6 +22,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only Qwen2-VL model compatible with HuggingFace weights.""" + import logging import re from functools import partial diff --git a/python/sglang/srt/models/qwen2_audio.py b/python/sglang/srt/models/qwen2_audio.py index 98f30636a..669db5a39 100644 --- a/python/sglang/srt/models/qwen2_audio.py +++ b/python/sglang/srt/models/qwen2_audio.py @@ -22,6 +22,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only Qwen2-Audio model compatible with HuggingFace weights.""" + import logging from typing import Any, Iterable, List, Optional, Tuple diff --git a/python/sglang/srt/models/qwen2_vl.py b/python/sglang/srt/models/qwen2_vl.py index 3002bbe14..94e6a48bd 100644 --- a/python/sglang/srt/models/qwen2_vl.py +++ b/python/sglang/srt/models/qwen2_vl.py @@ -22,6 +22,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Inference-only Qwen2-VL model compatible with HuggingFace weights.""" + import logging from functools import lru_cache, partial from typing import Iterable, List, Optional, Tuple, Type, TypedDict diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index ed2d6bd7e..413664176 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Inference-only Qwen3.5 model and Qwen3.5 MoE model compatible with HuggingFace weights.""" + import logging from functools import lru_cache from typing import Iterable, Optional, Set, Tuple, Union diff --git a/python/sglang/srt/models/qwen3_5_mtp.py b/python/sglang/srt/models/qwen3_5_mtp.py index c4b18a19a..069ba761e 100644 --- a/python/sglang/srt/models/qwen3_5_mtp.py +++ b/python/sglang/srt/models/qwen3_5_mtp.py @@ -13,6 +13,7 @@ # ============================================================================== """Inference-only Qwen3_5 MTP model.""" + import logging from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index ba89973f5..b565cbccc 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -366,8 +366,8 @@ class Qwen3GatedDeltaNet(nn.Module): # [b, sq, ng, (hn + hn + np/ng * hn + np/ng + np/ng)] # --> [b, sq, ng, hn], [b, sq, ng, hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng], [b, sq, ng, np/ng] - (query, key, value, z) = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2) - (b, a) = torch.split(mixed_ba, split_arg_list_ba, dim=2) + query, key, value, z = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2) + b, a = torch.split(mixed_ba, split_arg_list_ba, dim=2) # [b, sq, ng, np/ng * hn] -> [b, sq, np, hn] value = value.reshape(value.size(0), -1, self.head_v_dim) diff --git a/python/sglang/srt/models/qwen3_next_mtp.py b/python/sglang/srt/models/qwen3_next_mtp.py index aa0f8ec1e..9722f7e4a 100644 --- a/python/sglang/srt/models/qwen3_next_mtp.py +++ b/python/sglang/srt/models/qwen3_next_mtp.py @@ -13,6 +13,7 @@ # ============================================================================== """Inference-only Qwen3Next MTP Speculative Decoding.""" + import logging from typing import Iterable, Optional, Tuple diff --git a/python/sglang/srt/models/qwen3_omni_moe.py b/python/sglang/srt/models/qwen3_omni_moe.py index 5c80458b6..e0e82d03e 100644 --- a/python/sglang/srt/models/qwen3_omni_moe.py +++ b/python/sglang/srt/models/qwen3_omni_moe.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Inference-only Qwen3-VL model compatible with HuggingFace weights.""" + import math from typing import Iterable, List, Optional, Tuple diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 57c6a3973..2a8628dbc 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Inference-only Qwen3-VL model compatible with HuggingFace weights.""" + import logging import math import re diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 671566c46..f98460665 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Inference-only Qwen3-VL model compatible with HuggingFace weights.""" + import logging import re from functools import lru_cache diff --git a/python/sglang/srt/models/starcoder2.py b/python/sglang/srt/models/starcoder2.py index bbbcf8aeb..2ad4351e9 100644 --- a/python/sglang/srt/models/starcoder2.py +++ b/python/sglang/srt/models/starcoder2.py @@ -20,7 +20,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/starcoder2.py -""" PyTorch Starcoder2 model.""" +"""PyTorch Starcoder2 model.""" + from collections.abc import Iterable from typing import Optional, Tuple diff --git a/python/sglang/srt/models/transformers.py b/python/sglang/srt/models/transformers.py index 40e7edcaf..0ea9da14a 100644 --- a/python/sglang/srt/models/transformers.py +++ b/python/sglang/srt/models/transformers.py @@ -15,6 +15,7 @@ # Adapted from # https://github.com/vllm-project/vllm/blob/a1a2aaadb9122f05667140e39cf67e5736c8b6d6/vllm/model_executor/models/transformers.py """Wrapper around `transformers` models""" + import logging import re from typing import Iterable, Literal, Optional, Tuple, Union diff --git a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py index 07bc3e77d..8a5611679 100644 --- a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py @@ -13,6 +13,7 @@ # ============================================================================== """ViT CUDA Graph Runner class.""" + from __future__ import annotations from typing import Dict, Hashable, Tuple diff --git a/python/sglang/srt/multimodal/mm_utils.py b/python/sglang/srt/multimodal/mm_utils.py index bb7802f7f..2e402ccf5 100644 --- a/python/sglang/srt/multimodal/mm_utils.py +++ b/python/sglang/srt/multimodal/mm_utils.py @@ -27,6 +27,7 @@ LLaVA-NeXT : https://llava-vl.github.io/blog/2024-01-30-llava-next/ LLaVA-Onevision : https://arxiv.org/pdf/2408.03326 """ + import ast import itertools import math @@ -519,7 +520,7 @@ def run_dp_sharded_mrope_vision_model( # image_to_tp_rank = [0, 2, 1, 3] # gpu_sample_counts = [1, 3] # grouped_pixel_values_len = [1000, 350] - (image_to_tp_rank, gpu_sample_counts, grouped_pixel_values_len) = ( + image_to_tp_rank, gpu_sample_counts, grouped_pixel_values_len = ( get_dp_encoder_lb_assignment(patches_per_image, tp_size) ) diff --git a/python/sglang/srt/multimodal/processors/ernie45_vl.py b/python/sglang/srt/multimodal/processors/ernie45_vl.py index 7ec25015a..8b3633c59 100644 --- a/python/sglang/srt/multimodal/processors/ernie45_vl.py +++ b/python/sglang/srt/multimodal/processors/ernie45_vl.py @@ -15,7 +15,9 @@ from sglang.srt.models.ernie45_vl import Ernie4_5_VLMoeForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) from sglang.srt.utils import get_bool_env_var, is_npu, logger _is_npu = is_npu() diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index 45dc785fb..0dce4e956 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -7,7 +7,9 @@ from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) class Glm4vImageProcessor(SGLangBaseProcessor): diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py index bab6b28a6..d8bb9ceb3 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k25.py +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -8,7 +8,9 @@ from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) # Compatible with KimiVLForConditionalGeneration diff --git a/python/sglang/srt/multimodal/processors/kimi_vl.py b/python/sglang/srt/multimodal/processors/kimi_vl.py index 541ed5c9e..cd7cfe2fd 100644 --- a/python/sglang/srt/multimodal/processors/kimi_vl.py +++ b/python/sglang/srt/multimodal/processors/kimi_vl.py @@ -5,7 +5,9 @@ from sglang.srt.models.kimi_vl import KimiVLForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) # Compatible with KimiVLForConditionalGeneration diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index 9e7d0346f..4395654e4 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -26,7 +26,9 @@ from sglang.srt.models.qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) from sglang.utils import logger IMAGE_FACTOR = 28 diff --git a/python/sglang/srt/multimodal/processors/step3_vl.py b/python/sglang/srt/multimodal/processors/step3_vl.py index 658ba5882..d11d97787 100644 --- a/python/sglang/srt/multimodal/processors/step3_vl.py +++ b/python/sglang/srt/multimodal/processors/step3_vl.py @@ -15,7 +15,9 @@ from sglang.srt.models.step3_vl_10b import StepVLForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, ) -from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens +from sglang.srt.multimodal.processors.base_processor import ( + MultimodalSpecialTokens, +) ImageWithPatches = tuple[Image.Image, list[Image.Image], list[int] | None] diff --git a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py index 683271be5..cfdf62915 100644 --- a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py @@ -13,6 +13,7 @@ # ============================================================================== """ViT CUDA Graph Runner class.""" + from __future__ import annotations import inspect diff --git a/python/sglang/srt/parser/code_completion_parser.py b/python/sglang/srt/parser/code_completion_parser.py index 0067ac471..17bdd90b9 100644 --- a/python/sglang/srt/parser/code_completion_parser.py +++ b/python/sglang/srt/parser/code_completion_parser.py @@ -13,7 +13,6 @@ # ============================================================================== """Completion templates.""" - import dataclasses import logging from enum import auto diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index d61ddcaf9..415a44452 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -12,6 +12,7 @@ # limitations under the License. # ============================================================================== """Common utilities.""" + from __future__ import annotations import argparse diff --git a/python/sglang/srt/utils/weight_checker.py b/python/sglang/srt/utils/weight_checker.py index ca5c2e08e..3be16446e 100644 --- a/python/sglang/srt/utils/weight_checker.py +++ b/python/sglang/srt/utils/weight_checker.py @@ -117,7 +117,7 @@ def _random_like(t: torch.Tensor): def _postprocess_tensors( - raw: Dict[str, torch.Tensor] + raw: Dict[str, torch.Tensor], ) -> Iterable[Tuple[str, bool, torch.Tensor]]: from sglang.srt.debug_utils.dumper import get_tensor_info diff --git a/python/sglang/utils.py b/python/sglang/utils.py index cefa4bbe7..e2ae3aafb 100644 --- a/python/sglang/utils.py +++ b/python/sglang/utils.py @@ -531,15 +531,13 @@ def wait_for_server( headers={"Authorization": "Bearer None"}, ) time.sleep(5) - print_highlight( - """\n + print_highlight("""\n NOTE: Typically, the server runs in a separate terminal. In this notebook, we run the server and notebook code together, so their outputs are combined. To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. To reduce the log length, we set the log level to warning for the server, the default log level is info. We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance. - """ - ) + """) class TypeBasedDispatcher: diff --git a/scripts/ci/amd/test_rccl_multi_gpu.py b/scripts/ci/amd/test_rccl_multi_gpu.py index ffad64372..897780ab0 100755 --- a/scripts/ci/amd/test_rccl_multi_gpu.py +++ b/scripts/ci/amd/test_rccl_multi_gpu.py @@ -3,6 +3,7 @@ Simple RCCL test for multi-GPU communication. This test verifies that RCCL can initialize and communicate across multiple GPUs. """ + import os import sys diff --git a/scripts/ci_monitor/ci_analyzer.py b/scripts/ci_monitor/ci_analyzer.py index 409960f16..274e74842 100755 --- a/scripts/ci_monitor/ci_analyzer.py +++ b/scripts/ci_monitor/ci_analyzer.py @@ -618,7 +618,9 @@ class SGLangCIAnalyzer: if change_pct < -10: trend_indicator = f"📈 {change_pct:.1f}%" elif change_pct > 10: - trend_indicator = f"⚠️ 📉 +{change_pct:.1f}%" + trend_indicator = ( + f"⚠️ 📉 +{change_pct:.1f}%" + ) else: trend_indicator = f"➡️ {change_pct:+.1f}%" else: diff --git a/scripts/code_sync/copy_to_oss.py b/scripts/code_sync/copy_to_oss.py index d87a81dd7..96bc0af25 100644 --- a/scripts/code_sync/copy_to_oss.py +++ b/scripts/code_sync/copy_to_oss.py @@ -157,7 +157,9 @@ def get_oss_repo(dry_run): """ gh_token = os.getenv("GH_TOKEN") if not gh_token: - print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + print( + "⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation." + ) if not dry_run: return @@ -359,7 +361,9 @@ def create_pull_request(oss_root, branch_name, title, body, dry_run): """Create a pull request in the OSS repo using the GitHub CLI.""" gh_token = os.getenv("GH_TOKEN") if not gh_token: - print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + print( + "⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation." + ) if not dry_run: return diff --git a/scripts/playground/frontend_reasoning.ipynb b/scripts/playground/frontend_reasoning.ipynb index c2412f638..e9eaa1a7f 100644 --- a/scripts/playground/frontend_reasoning.ipynb +++ b/scripts/playground/frontend_reasoning.ipynb @@ -23,7 +23,6 @@ "from sglang.test.test_utils import is_in_ci\n", "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", "\n", - "\n", "if is_in_ci():\n", " from patch import launch_server_cmd\n", "else:\n", diff --git a/scripts/playground/replay_request_dump.py b/scripts/playground/replay_request_dump.py index 1881b1d68..e63c027aa 100644 --- a/scripts/playground/replay_request_dump.py +++ b/scripts/playground/replay_request_dump.py @@ -64,7 +64,7 @@ def read_records(files): def run_one_request_internal(record): - (req, output, replay_init_time, start_time, end_time, idx) = record + req, output, replay_init_time, start_time, end_time, idx = record time.sleep(max(0, (start_time - (time.time() - replay_init_time)) / args.speed)) if "completion_tokens" in output.get("meta_info", {}): diff --git a/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py b/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py index 04610b17c..5ad6f9fc2 100644 --- a/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py +++ b/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py @@ -14,7 +14,9 @@ from sglang.srt.layers.quantization.fp8_kernel import ( from sglang.srt.layers.quantization.fp8_kernel import ( per_token_group_quant_8bit as triton_per_token_group_quant_8bit, ) -from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_8bit +from sglang.srt.layers.quantization.fp8_kernel import ( + sglang_per_token_group_quant_8bit, +) from sglang.srt.utils import is_hip from sglang.srt.utils.bench_utils import bench_kineto diff --git a/sgl-kernel/csrc/attention/merge_attn_states.cu b/sgl-kernel/csrc/attention/merge_attn_states.cu index c719498b8..fca370f9e 100644 --- a/sgl-kernel/csrc/attention/merge_attn_states.cu +++ b/sgl-kernel/csrc/attention/merge_attn_states.cu @@ -176,8 +176,10 @@ void merge_attn_states_launcher( LAUNCH_MERGE_ATTN_STATES(scalar_t, NUM_THREADS); } -#define CALL_MERGE_ATTN_STATES_LAUNCHER(scalar_t) \ - { merge_attn_states_launcher(v_a, s_a, v_b, s_b, v_merged, s_merged); } +#define CALL_MERGE_ATTN_STATES_LAUNCHER(scalar_t) \ + { \ + merge_attn_states_launcher(v_a, s_a, v_b, s_b, v_merged, s_merged); \ + } void merge_state_v2( at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged) { diff --git a/sgl-kernel/csrc/common_extension_rocm.cc b/sgl-kernel/csrc/common_extension_rocm.cc index 495fb43a2..1c7265b2b 100644 --- a/sgl-kernel/csrc/common_extension_rocm.cc +++ b/sgl-kernel/csrc/common_extension_rocm.cc @@ -65,9 +65,9 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) { m.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg); // Deterministic all-reduce for ROCm - extern void deterministic_all_reduce_reg(int64_t _fa, torch::Tensor & inp, torch::Tensor & out); + extern void deterministic_all_reduce_reg(int64_t _fa, torch::Tensor& inp, torch::Tensor& out); extern void deterministic_all_reduce_unreg( - int64_t _fa, torch::Tensor & inp, torch::Tensor & reg_buffer, torch::Tensor & out); + int64_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, torch::Tensor& out); m.def("deterministic_all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()"); m.impl("deterministic_all_reduce_reg", torch::kCUDA, &deterministic_all_reduce_reg); diff --git a/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu index 53903e6af..dea405663 100644 --- a/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu +++ b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit_v2.cu @@ -452,7 +452,7 @@ void sgl_per_token_group_quant_8bit_v2( #define LAUNCH_KERNEL(GROUP_SIZE, T, DST_DTYPE) \ do { \ constexpr int THREADS_PER_SUBWARP = GROUP_SIZE / 16; \ - TORCH_CHECK(THREADS_PER_SUBWARP* INPUT_PRIMARY_VEC_NUM_BYTES == group_size * sizeof(T)); \ + TORCH_CHECK(THREADS_PER_SUBWARP * INPUT_PRIMARY_VEC_NUM_BYTES == group_size * sizeof(T)); \ \ using dst_dtype_info = DtypeInfo; \ CHECK_EQ(dst_dtype_info::MIN, min_8bit); \ diff --git a/sgl-kernel/python/sgl_kernel/gemm.py b/sgl-kernel/python/sgl_kernel/gemm.py index 4e23ebdd9..d98a68323 100644 --- a/sgl-kernel/python/sgl_kernel/gemm.py +++ b/sgl-kernel/python/sgl_kernel/gemm.py @@ -454,7 +454,7 @@ def scaled_fp4_experts_quant( input_tensor.ndim == 2 ), f"input.ndim needs to be == 2, but got {input_tensor.ndim}." if expert_map is not None: - (m, k) = input_tensor.shape + m, k = input_tensor.shape output_tensor_shape = (m * topk, k) input_tensor = shuffle_rows(input_tensor, expert_map, output_tensor_shape) m_numtopk, k = input_tensor.shape diff --git a/sgl-kernel/tests/test_per_token_group_quant_8bit.py b/sgl-kernel/tests/test_per_token_group_quant_8bit.py index 1f19af3f4..1f0190d99 100644 --- a/sgl-kernel/tests/test_per_token_group_quant_8bit.py +++ b/sgl-kernel/tests/test_per_token_group_quant_8bit.py @@ -13,7 +13,9 @@ from sgl_kernel.test_utils import ( from sglang.srt.layers.quantization.fp8_kernel import ( per_token_group_quant_8bit as triton_per_token_group_quant_8bit, ) -from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_8bit +from sglang.srt.layers.quantization.fp8_kernel import ( + sglang_per_token_group_quant_8bit, +) from sglang.srt.utils import get_bool_env_var, is_hip _is_hip = is_hip() diff --git a/test/manual/test_vlm_accuracy.py b/test/manual/test_vlm_accuracy.py index c722ed190..6e26c012a 100644 --- a/test/manual/test_vlm_accuracy.py +++ b/test/manual/test_vlm_accuracy.py @@ -1,5 +1,4 @@ -""" -""" +""" """ import unittest from typing import List, Optional @@ -195,7 +194,7 @@ class TestMiniCPMV2_6Logits(VisionLLMLogitsBase): "pixel_values": inputs.pixel_values, "tgt_sizes": inputs.tgt_sizes, } - (hf_output, _) = self.hf_model.get_vllm_embedding( + hf_output, _ = self.hf_model.get_vllm_embedding( model_inputs, ) hf_output = hf_output.squeeze(0) diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py index 3f4df13a5..6eeb1e04a 100644 --- a/test/registered/attention/test_triton_sliding_window.py +++ b/test/registered/attention/test_triton_sliding_window.py @@ -43,12 +43,9 @@ class TestSlidingWindowAttentionTriton(CustomTestCase): cls.short_context_prompt = "The capital of France is" # Test prompt longer than window size - cls.long_context_prompt = ( - """ + cls.long_context_prompt = """ Once upon a time, there was a mountain. In the mountain, there was a temple. In the temple, there was an old monk telling a story. The story was: - """ - * 100 - ) + """ * 100 cls.long_context_prompt += "\nNow, summarize the story in one sentence:" def _test_mmlu(self): diff --git a/test/registered/layers/test_fla_layernorm_guard.py b/test/registered/layers/test_fla_layernorm_guard.py index e0a89fde5..2d305cb62 100644 --- a/test/registered/layers/test_fla_layernorm_guard.py +++ b/test/registered/layers/test_fla_layernorm_guard.py @@ -10,7 +10,10 @@ import torch.nn.functional as F from sglang.srt.layers.attention.fla.layernorm_gated import ( _layer_norm_fwd as layer_norm_fwd, ) -from sglang.srt.layers.attention.fla.layernorm_gated import layernorm_fn, rms_norm_ref +from sglang.srt.layers.attention.fla.layernorm_gated import ( + layernorm_fn, + rms_norm_ref, +) from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( diff --git a/test/registered/quant/test_awq_dequant.py b/test/registered/quant/test_awq_dequant.py index 8f63a8243..3f9ac10e9 100644 --- a/test/registered/quant/test_awq_dequant.py +++ b/test/registered/quant/test_awq_dequant.py @@ -7,6 +7,7 @@ unittest version of the AWQ Triton kernel tests. Run with: python -m unittest test_awq_dequant.py """ + import unittest import torch diff --git a/test/srt/cpu/test_norm.py b/test/srt/cpu/test_norm.py index c118ab63e..434736a82 100644 --- a/test/srt/cpu/test_norm.py +++ b/test/srt/cpu/test_norm.py @@ -222,7 +222,7 @@ class TestLayerNorm(CustomTestCase): x = x + residual.to(torch.float32) residual = x.to(orig_dtype) - (variance, mean) = torch.var_mean(x, dim=-1, keepdim=True, correction=0) + variance, mean = torch.var_mean(x, dim=-1, keepdim=True, correction=0) x = (x - mean) * torch.rsqrt(variance + variance_epsilon) x = x.to(orig_dtype) * weight if residual is None: diff --git a/test/srt/cpu/test_qwen3.py b/test/srt/cpu/test_qwen3.py index 4c122c1bb..0ea95be52 100644 --- a/test/srt/cpu/test_qwen3.py +++ b/test/srt/cpu/test_qwen3.py @@ -39,8 +39,8 @@ def fix_query_key_value_ordering_reshape_cat( ] # [b, sq, ng, (hn + hn + np/ng * hn + np/ng + np/ng)] # --> [b, sq, ng, hn], [b, sq, ng, hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng * hn], [b, sq, ng, np/ng], [b, sq, ng, np/ng] - (query, key, value, z) = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2) - (b, a) = torch.split(mixed_ba, split_arg_list_ba, dim=2) + query, key, value, z = torch.split(mixed_qkvz, split_arg_list_qkvz, dim=2) + b, a = torch.split(mixed_ba, split_arg_list_ba, dim=2) # [b, sq, ng, np/ng * hn] -> [b, sq, np, hn] value = value.reshape(value.size(0), -1, head_v_dim)