Skip to content

Commit 61d26ff

Browse files
committed
refactor: Integrate circuit breaker and provider auth blocking in LLMRouter
1 parent 237d369 commit 61d26ff

1 file changed

Lines changed: 22 additions & 1 deletion

File tree

core/llm.py

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from typing import Any, Protocol
99

1010
from django.conf import settings
11+
from .resilience import circuit_breaker
1112

1213

1314
class LLMTask(str, Enum):
@@ -410,6 +411,21 @@ def generate(
410411
)
411412
continue
412413

414+
# Check circuit breaker
415+
circuit_name = f"llm:{adapter.name}:{adapter.model}"
416+
if not circuit_breaker.is_available(circuit_name):
417+
cooldown = circuit_breaker.cooldown_remaining(circuit_name)
418+
attempts.append(
419+
LLMAttempt(
420+
adapter.name,
421+
adapter.model,
422+
"cooldown",
423+
error="Provider circuit is open (cooling down after consecutive failures).",
424+
retry_after_seconds=cooldown,
425+
)
426+
)
427+
continue
428+
413429
cooldown_remaining = self.cooldown_remaining(adapter.name, adapter.model)
414430
if cooldown_remaining > 0:
415431
attempts.append(
@@ -433,6 +449,7 @@ def generate(
433449
try:
434450
text, usage = adapter.generate(request)
435451
latency_ms = int((time.perf_counter() - started) * 1000)
452+
circuit_breaker.record_success(circuit_name)
436453
attempts.append(
437454
LLMAttempt(
438455
adapter.name,
@@ -462,7 +479,10 @@ def generate(
462479
retry_after_seconds=retry_after,
463480
)
464481
)
465-
if exc.rate_limited or exc.retryable:
482+
circuit_breaker.record_failure(circuit_name, str(exc))
483+
if exc.auth_error:
484+
blocked_providers.add(adapter.name)
485+
elif exc.rate_limited or exc.retryable:
466486
cooldown = retry_after or getattr(settings, "LLM_PROVIDER_COOLDOWN_SECONDS", 90)
467487
self.cooldown_provider(adapter.name, adapter.model, cooldown)
468488
continue
@@ -477,6 +497,7 @@ def generate(
477497
error=str(exc),
478498
)
479499
)
500+
circuit_breaker.record_failure(circuit_name, str(exc))
480501
self.cooldown_provider(
481502
adapter.name,
482503
adapter.model,

0 commit comments

Comments
 (0)