From 40738f59cfba8f29ae78b7d2a5e99057b267dbff Mon Sep 17 00:00:00 2001 From: leavelet Date: Wed, 17 Jun 2026 20:06:55 +0000 Subject: [PATCH] Cache EnvField reads to avoid per-call os.getenv in the hot path EnvField.get() re-read os.environ (os.getenv + parse) on every call. The attention/compose hot path reads many flags per layer / per compose (nsa_backend ~11/layer, cp_shared_kv_runtime ~12/compose) -- thousands of os.getenv + Python-method calls per forward for values that are fixed startup config. Cache the parsed value on first read; invalidate on every mutation through the API (set / override / clear). The only env vars mutated at runtime to pass information are NOT EnvFields (SGLANG_TMP_NCCL_COMM_VALUE is read by C getenv; SGLANG_RUN_ID / SGLANG_DP_RANK via raw os.environ), so the cache does not affect them. Convert the two direct os.environ["SGLANG_*"]=... writes in server_args (SGLANG_VLM_CACHE_SIZE_MB, SGLANG_ENABLE_DETERMINISTIC_INFERENCE) to envs.X.set() so they invalidate the cache. Verified: cache + set/override/clear invalidation behavioral test passes; 331 mem_cache/parser/layers unit tests pass (5 pre-existing CUDA-device-assert failures unrelated). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/sglang/srt/environ.py | 18 ++++++++++++++++++ python/sglang/srt/server_args.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 12c42a3ca..edb8874d3 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -43,6 +43,14 @@ class EnvField: # NOTE: environ can only accept str values, so we need a flag to indicate # whether the env var is explicitly set to None. self._set_to_none = False + # Env vars are startup config that does not change mid-run, so the parsed + # value is cached after the first read (hot paths read flags per-layer / + # per-compose). Every mutation path through this API (set/override/clear) + # invalidates the cache. The only env vars mutated at runtime to pass + # information are NOT EnvFields (e.g. SGLANG_TMP_NCCL_COMM_VALUE, read via + # raw getenv), so they are unaffected by this cache. + self._cache_valid = False + self._cache = None def __set_name__(self, owner, name): assert EnvField._allow_set_name, "Usage like `a = envs.A` is not allowed" @@ -52,6 +60,13 @@ class EnvField: raise NotImplementedError() def get(self) -> Any: + if self._cache_valid: + return self._cache + self._cache = self._compute() + self._cache_valid = True + return self._cache + + def _compute(self) -> Any: value = os.getenv(self.name) # Explicitly set to None @@ -77,6 +92,7 @@ class EnvField: def set(self, value: Any): self._set_to_none = value is None os.environ[self.name] = str(value) + self._cache_valid = False @contextmanager def override(self, value: Any): @@ -90,10 +106,12 @@ class EnvField: else: os.environ.pop(self.name, None) self._set_to_none = backup_set_to_none + self._cache_valid = False def clear(self): os.environ.pop(self.name, None) self._set_to_none = False + self._cache_valid = False def __bool__(self): raise RuntimeError( diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 39adfcf3c..56e0b2c73 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -3606,9 +3606,9 @@ class ServerArgs: self.enable_deterministic_inference = True # For VLM - os.environ["SGLANG_VLM_CACHE_SIZE_MB"] = "0" + envs.SGLANG_VLM_CACHE_SIZE_MB.set("0") # TODO remove this environment variable as a whole - os.environ["SGLANG_ENABLE_DETERMINISTIC_INFERENCE"] = "1" + envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set("1") if self.enable_deterministic_inference: if self.enable_aiter_allreduce_fusion: