diff --git a/.env.example b/.env.example index 387829a2..fedbc10f 100644 --- a/.env.example +++ b/.env.example @@ -268,6 +268,15 @@ MAX_CONCURRENT_REQUESTS_PER_KEY_IFLOW=1 # EXHAUSTION_COOLDOWN_THRESHOLD_ANTIGRAVITY=300 # EXHAUSTION_COOLDOWN_THRESHOLD=300 # Global fallback for all providers +# Fallback cooldown controls for cross-provider fallback +# Multiplier applied to exhaustion threshold when fallback triggers +# FALLBACK_COOLDOWN_MULTIPLIER_ANTIGRAVITY=1.5 +# FALLBACK_COOLDOWN_MULTIPLIER=1.5 # Global fallback for all providers + +# Minimum fallback cooldown in seconds when fallback triggers +# FALLBACK_COOLDOWN_MIN_SECONDS_ANTIGRAVITY=60 +# FALLBACK_COOLDOWN_MIN_SECONDS=60 # Global fallback for all providers + # ------------------------------------------------------------------------------ # | [ADVANCED] Custom Caps | # ------------------------------------------------------------------------------ diff --git a/.gitignore b/.gitignore index 3711fdfd..76db01db 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,9 @@ dmypy.json # pytype static analyzer .pytype/ +# Local tooling +.opencode/ + # Cython debug symbols cython_debug/ test_proxy.py diff --git a/scripts/fallback_dry_run.py b/scripts/fallback_dry_run.py new file mode 100644 index 00000000..63371da8 --- /dev/null +++ b/scripts/fallback_dry_run.py @@ -0,0 +1,257 @@ +import asyncio +import asyncio +import json +import logging +from typing import Any, Dict + +try: + import litellm +except ModuleNotFoundError as exc: + print("Missing dependency: litellm. Activate venv and install requirements.") + raise SystemExit(1) from exc + +import os +import sys + +repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +src_root = os.path.join(repo_root, "src") +if src_root not in sys.path: + sys.path.insert(0, src_root) + +try: + from rotator_library.client import RotatingClient + from rotator_library.error_handler import NoAvailableKeysError +except ModuleNotFoundError as exc: + print("Missing rotator_library module. Run from repository root with venv active.") + raise SystemExit(1) from exc + + +class DummyProvider: + def __init__(self, model_name: str): + self._model_name = model_name + + async def get_models(self, credential: str, http_client: Any): + return [self._model_name] + + def has_custom_logic(self) -> bool: + return False + + + +async def _fake_acompletion(**kwargs: Any): + model = kwargs.get("model") + key = kwargs.get("_dry_run_key") + print(f"Dry run: acompletion attempt for {model} with {key}") + failure_budget = kwargs.get("_dry_run_failure_budget") or {} + remaining = failure_budget.get(key, 0) + if remaining > 0: + failure_budget[key] = remaining - 1 + print(f"Dry run: simulating rate limit for {model} with {key}") + raise litellm.RateLimitError("Dry run: simulated rate limit") + print(f"Dry run: returning success for {model} with {key}") + return { + "id": "dry-run", + "object": "chat.completion", + "created": 0, + "model": model or "unknown", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "dry-run"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +async def _fake_streaming_acompletion(**kwargs: Any): + model = kwargs.get("model") + if model and model.startswith("gemini_cli/"): + raise NoAvailableKeysError(f"Dry run: simulated exhaustion for {model}") + + async def _generator(): + yield { + "id": "dry-run", + "object": "chat.completion.chunk", + "created": 0, + "model": model or "unknown", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "dry"}} + ], + } + yield { + "id": "dry-run", + "object": "chat.completion.chunk", + "created": 0, + "model": model or "unknown", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + + return _generator() + + +def _configure_demo_provider(client: RotatingClient, provider: str, model: str): + client._provider_instances[provider] = DummyProvider(model) + client._provider_plugins[provider] = DummyProvider + + +async def run_demo() -> int: + logging.basicConfig(level=logging.INFO) + client = RotatingClient( + api_keys={ + "gemini_cli": [ + "dummy-gemini-key-1", + "dummy-gemini-key-2", + ], + "antigravity": ["dummy-antigravity-key-1"], + }, + configure_logging=False, + ) + + model_name = "gemini-pro" + primary_model = f"gemini_cli/{model_name}" + fallback_model = f"antigravity/{model_name}" + + _configure_demo_provider(client, "gemini_cli", primary_model) + _configure_demo_provider(client, "antigravity", fallback_model) + + client._model_list_cache["gemini_cli"] = [primary_model] + client._model_list_cache["antigravity"] = [fallback_model] + + client._convert_model_params_for_litellm = lambda **kwargs: kwargs + + last_key: Dict[str, str] = {} + + def _fake_get_provider_kwargs(**kwargs: Any) -> Dict[str, Any]: + payload = dict(kwargs) + payload["_dry_run_key"] = last_key.get("value") + payload["_dry_run_failure_budget"] = failure_budget + return payload + + client.all_providers.get_provider_kwargs = _fake_get_provider_kwargs + + failure_budget = { + "dummy-gemini-key-1": 1, + "dummy-gemini-key-2": 1, + "dummy-antigravity-key-1": 0, + } + + async def _fake_release_key(key: str, model: str): + return None + + key_cursor: Dict[str, int] = {} + + async def _fake_acquire_key(**kwargs: Any): + available_keys = kwargs.get("available_keys") or [] + if not available_keys: + return "dummy-key" + key = available_keys[key_cursor.get("idx", 0) % len(available_keys)] + key_cursor["idx"] = key_cursor.get("idx", 0) + 1 + last_key["value"] = key + return key + + async def _fake_availability_stats( + creds: Any, model: str, credential_priorities: Any + ) -> Dict[str, int]: + return {"available": len(creds), "on_cooldown": 0, "fair_cycle_excluded": 0} + + async def _fake_record_success(*_args: Any, **_kwargs: Any): + return None + + async def _fake_record_failure(*_args: Any, **_kwargs: Any): + return None + + async def _fake_acquire_key_proxy(*args: Any, **kwargs: Any): + return await _fake_acquire_key(**kwargs) + + original_acquire_key = client.usage_manager.acquire_key + original_release_key = client.usage_manager.release_key + original_availability = client.usage_manager.get_credential_availability_stats + original_record_success = client.usage_manager.record_success + original_record_failure = client.usage_manager.record_failure + client.usage_manager.acquire_key = _fake_acquire_key_proxy + client.usage_manager.release_key = _fake_release_key + client.usage_manager.get_credential_availability_stats = _fake_availability_stats + client.usage_manager.record_success = _fake_record_success + client.usage_manager.record_failure = _fake_record_failure + + async def _fake_streaming_impl( + request: Any, pre_request_callback: Any = None, **kwargs: Any + ): + model = kwargs.get("model") + print(f"Dry run: streaming attempt for {model}") + try: + stream_source = await _fake_streaming_acompletion(**kwargs) + except NoAvailableKeysError: + if model and model.startswith("gemini_cli/"): + fallback_model = f"antigravity/{model.split('/', 1)[1]}" + else: + fallback_model = "antigravity/gemini-pro" + print( + f"Dry run: streaming fallback from {model} to {fallback_model}" + ) + fallback_kwargs = dict(kwargs) + fallback_kwargs["model"] = fallback_model + stream_source = await _fake_streaming_acompletion(**fallback_kwargs) + + async for chunk in stream_source: + yield f"data: {json.dumps(chunk)}\n\n" + yield "data: [DONE]\n\n" + + original_streaming_impl = client._streaming_acompletion_with_retry + client._streaming_acompletion_with_retry = _fake_streaming_impl + + original_acompletion = litellm.acompletion + litellm.acompletion = _fake_acompletion + try: + print("Dry run: triggering non-streaming fallback...") + try: + response = await client._execute_with_retry( + litellm.acompletion, request=None, model=primary_model + ) + if isinstance(response, dict): + print( + "Dry run: non-streaming result model = " + f"{response.get('model')}" + ) + except NoAvailableKeysError: + print("Non-streaming: fallback exhausted (unexpected in dry run)") + + remaining = await client.cooldown_manager.get_cooldown_remaining("gemini_cli") + print(f"Primary cooldown remaining: {remaining:.1f}s") + finally: + litellm.acompletion = original_acompletion + + try: + print("Dry run: triggering streaming fallback...") + stream = client._streaming_acompletion_with_retry( + request=None, + model=primary_model, + stream=True, + messages=[{"role": "user", "content": "ping"}], + ) + saw_done = False + async for chunk in stream: + payload = chunk.strip() + if payload: + print(payload) + if payload.endswith("[DONE]"): + saw_done = True + break + if not saw_done: + print("Streaming: no completion emitted (unexpected in dry run)") + finally: + client._streaming_acompletion_with_retry = original_streaming_impl + + client.usage_manager.acquire_key = original_acquire_key + client.usage_manager.release_key = original_release_key + client.usage_manager.get_credential_availability_stats = original_availability + client.usage_manager.record_success = original_record_success + client.usage_manager.record_failure = original_record_failure + + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(run_demo())) diff --git a/src/proxy_app/settings_tool.py b/src/proxy_app/settings_tool.py index 1194432b..725c3b4c 100644 --- a/src/proxy_app/settings_tool.py +++ b/src/proxy_app/settings_tool.py @@ -490,6 +490,17 @@ def remove_multiplier(self, provider: str, priority: int): "default": ANTIGRAVITY_DEFAULT_OAUTH_PORT, "description": "Local port for OAuth callback server during authentication", }, + # Cross-provider fallback tuning (extend for additional providers if needed) + "FALLBACK_COOLDOWN_MULTIPLIER_ANTIGRAVITY": { + "type": "float", + "default": 1.5, + "description": "Multiplier applied to exhaustion cooldown when fallback triggers", + }, + "FALLBACK_COOLDOWN_MIN_SECONDS_ANTIGRAVITY": { + "type": "int", + "default": 60, + "description": "Minimum fallback cooldown duration after switching providers", + }, } # Gemini CLI provider environment variables @@ -539,6 +550,17 @@ def remove_multiplier(self, provider: str, priority: int): "default": GEMINI_CLI_DEFAULT_OAUTH_PORT, "description": "Local port for OAuth callback server during authentication", }, + # Cross-provider fallback tuning (extend for additional providers if needed) + "FALLBACK_COOLDOWN_MULTIPLIER_GEMINI_CLI": { + "type": "float", + "default": 1.5, + "description": "Multiplier applied to exhaustion cooldown when fallback triggers", + }, + "FALLBACK_COOLDOWN_MIN_SECONDS_GEMINI_CLI": { + "type": "int", + "default": 60, + "description": "Minimum fallback cooldown duration after switching providers", + }, } # iFlow provider environment variables diff --git a/src/rotator_library/client.py b/src/rotator_library/client.py index a0ec4dfa..b83a162e 100644 --- a/src/rotator_library/client.py +++ b/src/rotator_library/client.py @@ -49,6 +49,8 @@ DEFAULT_ROTATION_TOLERANCE, DEFAULT_FAIR_CYCLE_DURATION, DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD, + DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER, + DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS, DEFAULT_SEQUENTIAL_FALLBACK_MULTIPLIER, ) @@ -361,6 +363,29 @@ def __init__( f"Invalid EXHAUSTION_COOLDOWN_THRESHOLD: {global_threshold_str}. Using default {DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD}." ) + # Fallback cooldown settings + fallback_cooldown_multiplier: Dict[str, float] = {} + fallback_cooldown_min_seconds: Dict[str, int] = {} + global_multiplier_str = os.getenv("FALLBACK_COOLDOWN_MULTIPLIER") + global_multiplier = DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER + if global_multiplier_str: + try: + global_multiplier = float(global_multiplier_str) + except ValueError: + lib_logger.warning( + f"Invalid FALLBACK_COOLDOWN_MULTIPLIER: {global_multiplier_str}. Using default {DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER}." + ) + + global_min_seconds_str = os.getenv("FALLBACK_COOLDOWN_MIN_SECONDS") + global_min_seconds = DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS + if global_min_seconds_str: + try: + global_min_seconds = int(global_min_seconds_str) + except ValueError: + lib_logger.warning( + f"Invalid FALLBACK_COOLDOWN_MIN_SECONDS: {global_min_seconds_str}. Using default {DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS}." + ) + for provider in self.all_credentials.keys(): provider_class = self._provider_plugins.get(provider) @@ -386,6 +411,46 @@ def __init__( # Use global threshold if set and different from default exhaustion_cooldown_threshold[provider] = global_threshold + # Fallback cooldown multiplier + multiplier_key = f"FALLBACK_COOLDOWN_MULTIPLIER_{provider.upper()}" + multiplier_val = os.getenv(multiplier_key) + if multiplier_val is not None: + try: + fallback_cooldown_multiplier[provider] = float(multiplier_val) + except ValueError: + lib_logger.warning( + f"Invalid {multiplier_key}: {multiplier_val}. Must be float." + ) + elif provider_class and hasattr( + provider_class, "default_fallback_cooldown_multiplier" + ): + default_multiplier = provider_class.default_fallback_cooldown_multiplier + if default_multiplier != DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER: + fallback_cooldown_multiplier[provider] = default_multiplier + elif global_multiplier != DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER: + fallback_cooldown_multiplier[provider] = global_multiplier + + # Fallback cooldown minimum seconds + min_seconds_key = f"FALLBACK_COOLDOWN_MIN_SECONDS_{provider.upper()}" + min_seconds_val = os.getenv(min_seconds_key) + if min_seconds_val is not None: + try: + fallback_cooldown_min_seconds[provider] = int(min_seconds_val) + except ValueError: + lib_logger.warning( + f"Invalid {min_seconds_key}: {min_seconds_val}. Must be integer." + ) + elif provider_class and hasattr( + provider_class, "default_fallback_cooldown_min_seconds" + ): + default_min_seconds = ( + provider_class.default_fallback_cooldown_min_seconds + ) + if default_min_seconds != DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS: + fallback_cooldown_min_seconds[provider] = default_min_seconds + elif global_min_seconds != DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS: + fallback_cooldown_min_seconds[provider] = global_min_seconds + # Log fair cycle configuration for provider, enabled in fair_cycle_enabled.items(): if not enabled: @@ -398,6 +463,16 @@ def __init__( for provider, cross_tier in fair_cycle_cross_tier.items(): if cross_tier: lib_logger.info(f"Provider '{provider}' fair cycle cross-tier: enabled") + for provider, multiplier in fallback_cooldown_multiplier.items(): + if multiplier != DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER: + lib_logger.info( + f"Provider '{provider}' fallback cooldown multiplier: {multiplier}" + ) + for provider, min_seconds in fallback_cooldown_min_seconds.items(): + if min_seconds != DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS: + lib_logger.info( + f"Provider '{provider}' fallback cooldown min seconds: {min_seconds}" + ) # Build custom caps configuration # Format: CUSTOM_CAP_{PROVIDER}_T{TIER}_{MODEL_OR_GROUP}= @@ -491,6 +566,9 @@ def __init__( else: resolved_usage_path = self.data_dir / "key_usage.json" + self.exhaustion_cooldown_threshold = exhaustion_cooldown_threshold + self.fallback_cooldown_multiplier = fallback_cooldown_multiplier + self.fallback_cooldown_min_seconds = fallback_cooldown_min_seconds self.usage_manager = UsageManager( file_path=resolved_usage_path, rotation_tolerance=rotation_tolerance, @@ -516,6 +594,13 @@ def __init__( self.enable_request_logging = enable_request_logging self.model_definitions = ModelDefinitions() + # [NEW] Provider Compatibility Map for Cross-Provider Fallback + # Defines bidirectional compatibility between providers for smart model fallback + self.provider_compatibility = { + "gemini_cli": "antigravity", + "antigravity": "gemini_cli", + } + # Store and validate max concurrent requests per key self.max_concurrent_requests_per_key = max_concurrent_requests_per_key or {} # Validate all values are >= 1 @@ -1989,6 +2074,26 @@ async def _execute_with_retry( # Log concise summary for server logs lib_logger.error(error_accumulator.build_log_message()) + # [Cross-Provider Fallback] + # If all credentials failed, try fallback provider if available + fallback_model = await self._get_fallback_model(model) + # Only fallback if we haven't already (prevent infinite recursion) + if fallback_model and not kwargs.get("_is_fallback_attempt"): + fallback_provider = fallback_model.split("/", 1)[0] + lib_logger.warning( + f"Cross-Provider Fallback: {model} exhausted (all keys failed). " + f"Switching to {fallback_model}." + ) + await self._apply_fallback_cooldown(provider, fallback_provider) + # Update model and mark as fallback attempt + kwargs["model"] = fallback_model + kwargs["_is_fallback_attempt"] = True + + # Recursive call with fallback model + return await self._execute_with_retry( + api_call, request, pre_request_callback, **kwargs + ) + # Return the structured error response for the client return error_accumulator.build_client_error_response() @@ -2794,6 +2899,24 @@ async def _streaming_acompletion_with_retry( # Log concise summary for server logs lib_logger.error(error_accumulator.build_log_message()) + # [Cross-Provider Fallback] + fallback_model = await self._get_fallback_model(model) + if fallback_model and not kwargs.get("_is_fallback_attempt"): + fallback_provider = fallback_model.split("/", 1)[0] + lib_logger.warning( + f"Cross-Provider Fallback: {model} exhausted (all keys failed). " + f"Switching to {fallback_model}." + ) + await self._apply_fallback_cooldown(provider, fallback_provider) + kwargs["model"] = fallback_model + kwargs["_is_fallback_attempt"] = True + + async for chunk in self._streaming_acompletion_with_retry( + request, pre_request_callback, **kwargs + ): + yield chunk + return + # Build structured error response for client error_response = error_accumulator.build_client_error_response() error_data = error_response @@ -2815,6 +2938,25 @@ async def _streaming_acompletion_with_retry( yield "data: [DONE]\n\n" except NoAvailableKeysError as e: + # [Cross-Provider Fallback] + fallback_model = await self._get_fallback_model(model) + if fallback_model and not kwargs.get("_is_fallback_attempt"): + fallback_provider = fallback_model.split("/", 1)[0] + lib_logger.warning( + f"Cross-Provider Fallback: {model} exhausted (NoAvailableKeysError). " + f"Switching to {fallback_model}." + ) + await self._apply_fallback_cooldown(provider, fallback_provider) + kwargs["model"] = fallback_model + kwargs["_is_fallback_attempt"] = True + + # Delegate to new stream generator + async for chunk in self._streaming_acompletion_with_retry( + request, pre_request_callback, **kwargs + ): + yield chunk + return + lib_logger.error( f"A streaming request failed because no keys were available within the time budget: {e}" ) @@ -2836,6 +2978,91 @@ async def _streaming_acompletion_with_retry( yield f"data: {json.dumps(error_data)}\n\n" yield "data: [DONE]\n\n" + async def _get_fallback_model(self, model: str) -> Optional[str]: + """ + Resolve a fallback model from a compatible provider if available. + Uses smart ID matching (e.g. gemini_cli/gemini-pro -> antigravity/gemini-pro). + """ + if "/" not in model: + return None + + provider, model_name = model.split("/", 1) + + # Check if we have a compatible fallback provider + fallback_provider = self.provider_compatibility.get(provider) + if not fallback_provider: + return None + + # Check if we have credentials for the fallback provider + # We need to check both API keys and OAuth credentials + has_creds = ( + fallback_provider in self.api_keys + or fallback_provider in self.oauth_credentials + ) + + if not has_creds: + return None + + fallback_model = f"{fallback_provider}/{model_name}" + if self._is_model_ignored(fallback_provider, fallback_model): + lib_logger.info( + f"Fallback model {fallback_model} is ignored for provider {fallback_provider}." + ) + return None + + if fallback_provider in self.whitelist_models and not self._is_model_whitelisted( + fallback_provider, fallback_model + ): + lib_logger.info( + f"Fallback model {fallback_model} is not whitelisted for provider {fallback_provider}." + ) + return None + + available_models = await self.get_available_models(fallback_provider) + if available_models: + if fallback_model not in available_models and model_name not in available_models: + if not any( + candidate.endswith(f"/{model_name}") + for candidate in available_models + ): + lib_logger.info( + f"Fallback model {fallback_model} is not available for provider {fallback_provider}." + ) + return None + else: + lib_logger.info( + f"Fallback provider {fallback_provider} reported no available models." + ) + return None + + # Construct fallback model ID + # We assume model names (suffixes) are consistent across compatible providers + return fallback_model + + async def _apply_fallback_cooldown( + self, primary_provider: str, fallback_provider: str + ) -> None: + if not primary_provider or primary_provider == fallback_provider: + return + + cooldown_threshold = self.exhaustion_cooldown_threshold.get( + primary_provider, DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD + ) + cooldown_multiplier = self.fallback_cooldown_multiplier.get( + primary_provider, DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER + ) + cooldown_min_seconds = self.fallback_cooldown_min_seconds.get( + primary_provider, DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS + ) + cooldown_seconds = max( + int(cooldown_threshold * cooldown_multiplier), + cooldown_min_seconds, + ) + await self.cooldown_manager.start_cooldown(primary_provider, cooldown_seconds) + lib_logger.info( + f"Applied fallback cooldown for {primary_provider}: {cooldown_seconds}s." + ) + def acompletion( self, request: Optional[Any] = None, diff --git a/src/rotator_library/config/__init__.py b/src/rotator_library/config/__init__.py index beacfd33..daf6c3cb 100644 --- a/src/rotator_library/config/__init__.py +++ b/src/rotator_library/config/__init__.py @@ -19,6 +19,8 @@ DEFAULT_FAIR_CYCLE_CROSS_TIER, DEFAULT_FAIR_CYCLE_DURATION, DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD, + DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER, + DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS, # Custom Caps DEFAULT_CUSTOM_CAP_COOLDOWN_MODE, DEFAULT_CUSTOM_CAP_COOLDOWN_VALUE, @@ -45,6 +47,8 @@ "DEFAULT_FAIR_CYCLE_CROSS_TIER", "DEFAULT_FAIR_CYCLE_DURATION", "DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD", + "DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER", + "DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS", # Custom Caps "DEFAULT_CUSTOM_CAP_COOLDOWN_MODE", "DEFAULT_CUSTOM_CAP_COOLDOWN_VALUE", diff --git a/src/rotator_library/config/defaults.py b/src/rotator_library/config/defaults.py index 73c9cacc..d8a432d8 100644 --- a/src/rotator_library/config/defaults.py +++ b/src/rotator_library/config/defaults.py @@ -83,6 +83,14 @@ # Global fallback: EXHAUSTION_COOLDOWN_THRESHOLD= DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD: int = 300 # 5 minutes +# Cross-provider fallback cooldown controls +# Apply a short provider-wide cooldown to the primary provider after fallback. +# Override: FALLBACK_COOLDOWN_MULTIPLIER_{PROVIDER}= +# Override: FALLBACK_COOLDOWN_MIN_SECONDS_{PROVIDER}= +# Global fallback: FALLBACK_COOLDOWN_MULTIPLIER / FALLBACK_COOLDOWN_MIN_SECONDS +DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER: float = 1.5 +DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS: int = 60 + # ============================================================================= # CUSTOM CAPS DEFAULTS # ============================================================================= diff --git a/src/rotator_library/providers/provider_interface.py b/src/rotator_library/providers/provider_interface.py index 3dabd69d..60a9d094 100644 --- a/src/rotator_library/providers/provider_interface.py +++ b/src/rotator_library/providers/provider_interface.py @@ -27,6 +27,8 @@ DEFAULT_FAIR_CYCLE_CROSS_TIER, DEFAULT_FAIR_CYCLE_DURATION, DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD, + DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER, + DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS, ) @@ -174,6 +176,16 @@ class ProviderInterface(ABC): # Global fallback: EXHAUSTION_COOLDOWN_THRESHOLD= default_exhaustion_cooldown_threshold: int = DEFAULT_EXHAUSTION_COOLDOWN_THRESHOLD + # Fallback cooldown controls + # Used to temporarily pause a primary provider after switching to a fallback. + # Can be overridden via env: + # FALLBACK_COOLDOWN_MULTIPLIER_{PROVIDER}= + # FALLBACK_COOLDOWN_MIN_SECONDS_{PROVIDER}= + # Global fallback: + # FALLBACK_COOLDOWN_MULTIPLIER / FALLBACK_COOLDOWN_MIN_SECONDS + default_fallback_cooldown_multiplier: float = DEFAULT_FALLBACK_COOLDOWN_MULTIPLIER + default_fallback_cooldown_min_seconds: int = DEFAULT_FALLBACK_COOLDOWN_MIN_SECONDS + # ========================================================================= # CUSTOM CAPS - Override in subclass # =========================================================================