From 1a83b2c15d6a890754e663063b675b7e99270929 Mon Sep 17 00:00:00 2001 From: xdtbynd <231153518+xdtbynd@users.noreply.github.com> Date: Tue, 24 Feb 2026 18:50:08 +0800 Subject: [PATCH] fix: fix the bug blocking completion template application (#17010) Co-authored-by: xdtbynd Co-authored-by: cy Co-authored-by: sglang-npu-bot --- .../sglang/srt/managers/template_manager.py | 3 + .../srt/parser/code_completion_parser.py | 13 ++- .../ascend/basic_function/deepseek_coder.json | 7 ++ .../test_ascend_fim_completion.py | 99 +++++++++++++++++++ 4 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 test/registered/ascend/basic_function/deepseek_coder.json create mode 100644 test/registered/ascend/basic_function/test_ascend_fim_completion.py diff --git a/python/sglang/srt/managers/template_manager.py b/python/sglang/srt/managers/template_manager.py index dd9bbcc55..996758477 100644 --- a/python/sglang/srt/managers/template_manager.py +++ b/python/sglang/srt/managers/template_manager.py @@ -30,6 +30,7 @@ from sglang.srt.parser.code_completion_parser import ( FimPosition, completion_template_exists, register_completion_template, + set_completion_template, ) from sglang.srt.parser.conversation import ( Conversation, @@ -199,6 +200,8 @@ class TemplateManager: else: self._completion_template_name = completion_template_arg + set_completion_template(self._completion_template_name) + def initialize_templates( self, tokenizer_manager: TokenizerManager, diff --git a/python/sglang/srt/parser/code_completion_parser.py b/python/sglang/srt/parser/code_completion_parser.py index 17bdd90b9..510f74468 100644 --- a/python/sglang/srt/parser/code_completion_parser.py +++ b/python/sglang/srt/parser/code_completion_parser.py @@ -15,15 +15,16 @@ import dataclasses import logging -from enum import auto +from enum import Enum, auto +from typing import Optional from sglang.srt.entrypoints.openai.protocol import CompletionRequest logger = logging.getLogger(__name__) -completion_template_name = None +completion_template_name: Optional[str] = None -class FimPosition: +class FimPosition(Enum): """Position of fim middle token.""" MIDDLE = auto() @@ -68,6 +69,12 @@ def completion_template_exists(template_name: str) -> bool: return template_name in completion_templates +def set_completion_template(template_name: str) -> None: + global completion_template_name + if completion_template_name is None: + completion_template_name = template_name + + def is_completion_template_defined() -> bool: global completion_template_name return completion_template_name is not None diff --git a/test/registered/ascend/basic_function/deepseek_coder.json b/test/registered/ascend/basic_function/deepseek_coder.json new file mode 100644 index 000000000..96cf02178 --- /dev/null +++ b/test/registered/ascend/basic_function/deepseek_coder.json @@ -0,0 +1,7 @@ +{ + "name": "deepseek_coder", + "fim_begin_token": "<｜fim▁begin｜>", + "fim_middle_token": "<｜fim▁hole｜>", + "fim_end_token": "<｜fim▁end｜>", + "fim_position": "MIDDLE" +} diff --git a/test/registered/ascend/basic_function/test_ascend_fim_completion.py b/test/registered/ascend/basic_function/test_ascend_fim_completion.py new file mode 100644 index 000000000..4ba0e7c94 --- /dev/null +++ b/test/registered/ascend/basic_function/test_ascend_fim_completion.py @@ -0,0 +1,99 @@ +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.srt.utils.hf_transformers_utils import get_tokenizer +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) + + +class TestFimCompletion(CustomTestCase): + """Testcase：Verify set --completion-template, the model's FIM (Fill-in-the-Middle) completion function work correctly. + + [Test Category] Parameter + [Test Target] --completion-template + """ + + model = "/root/.cache/modelscope/hub/models/deepseek-ai/deepseek-coder-1.3b-base" + other_args = [ + "--completion-template", + "deepseek_coder", + "--attention-backend", + "ascend", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + ] + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=cls.other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_fim_completion(self, number_of_completion): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "function sum(a: number, b: number): number{\n" + suffix = "}" + + prompt_input = self.tokenizer.encode(prompt) + self.tokenizer.encode(suffix) + num_prompt_tokens = len(prompt_input) + 2 + + response = client.completions.create( + model=self.model, + prompt=prompt, + suffix=suffix, + temperature=0.3, + max_tokens=32, + stream=False, + n=number_of_completion, + ) + assert len(response.choices) == number_of_completion + assert response.id + assert response.created + assert response.object == "text_completion" + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_fim_completion(self): + for number_of_completion in [1, 3]: + self.run_fim_completion(number_of_completion) + + +class TestFimCompletionJson(TestFimCompletion): + other_args = [ + "--completion-template", + "./deepseek_coder.json", + "--attention-backend", + "ascend", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + ] + + +if __name__ == "__main__": + unittest.main()