Skip to content

llm_factory: cache parsed llm_config per profile #671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
neoneye merged 1 commit into from
May 5, 2026
Merged

llm_factory: cache parsed llm_config per profile #671

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion worker_plan/worker_plan_internal/llm_factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,15 @@ def _resolve_model_profile(model_profile: Optional[ModelProfileEnum | str]) -> M
return resolve_model_profile_from_env()


_llm_config_cache: dict[ModelProfileEnum, PlanExeLLMConfig] = {}


def _load_llm_config(model_profile: Optional[ModelProfileEnum | str]) -> PlanExeLLMConfig:
resolved_profile = _resolve_model_profile(model_profile)
cached = _llm_config_cache.get(resolved_profile)
if cached is not None:
return cached

planexe_llmconfig = PlanExeLLMConfig.load(model_profile=resolved_profile)
try:
from worker_plan_internal.llm_util.model_pricing import load_pricing_from_llm_config
Expand All @@ -64,6 +71,8 @@ def _load_llm_config(model_profile: Optional[ModelProfileEnum | str]) -> PlanExe
install_anthropic_usage_hook()
except Exception:
logger.debug("Failed to install Anthropic usage hook", exc_info=True)

_llm_config_cache[resolved_profile] = planexe_llmconfig
return planexe_llmconfig


Expand Down Expand Up @@ -230,7 +239,9 @@ def get_llm(llm_name: Optional[str] = None, model_profile: Optional[ModelProfile

config = planexe_llmconfig.llm_config_dict[llm_name]
class_name = config.get("class")
arguments = config.get("arguments", {})
# Copy before mutating — _load_llm_config caches the parsed config
# across calls, so we must not write per-call kwargs back into it.
arguments = dict(config.get("arguments", {}))

# Override with any kwargs passed to get_llm()
arguments.update(kwargs)
Expand Down