Skip to content

Commit 7ebe354

Browse files
authored
🐛 use eager compile (#488)
# Description In some environments the default of `inductor` here is breaking vllm where a compiler isn't available. This PR makes the simple compile backend configurable, and defaults to `eager` for portability. Signed-off-by: Joe Runde <[email protected]>
1 parent f1944fc commit 7ebe354

File tree

2 files changed

+11
-3
lines changed

2 files changed

+11
-3
lines changed

vllm_spyre/envs.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
VLLM_SPYRE_WORKER_LOG_REDIRECT_DIR: str = ""
2424
VLLM_SPYRE_GLOO_TIMEOUT_MINUTES: int = 60
2525
VLLM_SPYRE_REQUIRE_PRECOMPILED_DECODERS: bool = False
26+
VLLM_SPYRE_SIMPLE_COMPILE_BACKEND: str = "eager"
2627

2728
logger = init_logger(__name__)
2829

@@ -143,7 +144,14 @@ def _backend_backwards_compat() -> str:
143144
# disable compilation for decoders
144145
"VLLM_SPYRE_REQUIRE_PRECOMPILED_DECODERS":
145146
lambda: bool(int(os.getenv("VLLM_SPYRE_REQUIRE_PRECOMPILED_DECODERS", "0"))
146-
)
147+
),
148+
149+
# Simple compile backend for some dynamically compiled operations, like
150+
# gathering logprobs in the sampler.
151+
# Defaults to eager, iductor can be used if python headers and a compiler
152+
# are available.
153+
"VLLM_SPYRE_SIMPLE_COMPILE_BACKEND":
154+
lambda: os.getenv("VLLM_SPYRE_SIMPLE_COMPILE_BACKEND", "eager"),
147155
}
148156
# --8<-- [end:env-vars-definition]
149157

vllm_spyre/platform.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ class SpyrePlatform(Platform):
6565
_num_spyre_blocks_override: int = -1 # override num of KV cache blocks
6666
_config: VllmConfig = None
6767

68-
# TODO: see if this needs to be set
68+
# Backend for dynamic compilation ops
6969
# See vllm batched_count_greater_than method
70-
# simple_compile_backend: str = "eager"
70+
simple_compile_backend: str = envs_spyre.VLLM_SPYRE_SIMPLE_COMPILE_BACKEND
7171

7272
# Needed by vllm/model_executor/layers/pooler.py:562
7373
current_stream = lambda _: _StreamPlaceholder()

0 commit comments

Comments
 (0)