Skip to content

Commit 895c981

Browse files
hydropixclaude
hydropix
and
claude
committed
fix(#217): rate-limit key rotation no longer consumes retry attempts
Key rotation on a 429 burned one MAX_TRANSLATION_ATTEMPTS attempt per rotation, so with the default of 2 attempts a 3-key pool never tried key 3 and never raised RateLimitError: chunks failed silently (untranslated output) in exactly the multi-key setups the pool was built for, and the pipeline auto-pause never engaged. Separate the budgets: providers now run a while loop where only transient errors (timeout, 5xx, bad JSON) increment the attempt counter, while 429s increment their own rate_limit_events counter. handle_rate_limit() gets an explicit budget of pool_size + max_attempts - 1 rate-limit events per request (one rotation per key plus the historical single-key sleep-waits), past which it raises RateLimitError even if a key has recovered, guarding against an unbounded rotate loop. Single-key behavior is unchanged (sleep once, then raise). Applies to all six pooled providers: gemini, openai-compatible, mistral, deepseek, poe, openrouter. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent b81acf7 commit 895c981

9 files changed

Lines changed: 268 additions & 75 deletions

File tree

‎src/core/llm/key_pool.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,26 @@ class _KeyState:
3232
class KeyPool:
3333
"""Round-robin pool of API keys with per-key throttle tracking.
3434
35-
Typical usage from a provider:
35+
Typical usage from a provider (note the two separate counters — rotating
36+
on 429 must not consume a transient-retry attempt, see issue #217):
3637
37-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
38+
attempt = 0
39+
rate_limit_events = 0
40+
while attempt < MAX_TRANSLATION_ATTEMPTS:
3841
current_key = await self._key_pool.acquire()
3942
headers = {"Authorization": f"Bearer {current_key}"}
4043
try:
4144
response = await client.post(url, headers=headers, ...)
4245
...
4346
except httpx.HTTPStatusError as e:
4447
if e.response.status_code == 429:
45-
await handle_rate_limit(self._key_pool, current_key, ...)
48+
rate_limit_events += 1
49+
await handle_rate_limit(
50+
self._key_pool, current_key, e.response.headers,
51+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
52+
)
4653
continue
54+
attempt += 1
4755
...
4856
"""
4957

‎src/core/llm/providers/deepseek.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,11 @@ async def generate(
222222
payload["thinking"] = {"type": "disabled"}
223223

224224
client = await self._get_client()
225-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
225+
# 429s have their own budget (rate_limit_events): rotating to a spare
226+
# key must not consume a transient-retry attempt (issue #217).
227+
attempt = 0
228+
rate_limit_events = 0
229+
while attempt < MAX_TRANSLATION_ATTEMPTS:
226230
current_key = await self._key_pool.acquire()
227231
headers = {
228232
"Authorization": f"Bearer {current_key}",
@@ -241,9 +245,10 @@ async def generate(
241245
raise ValueError("Invalid DeepSeek API key")
242246

243247
if response.status_code == 429:
248+
rate_limit_events += 1
244249
await handle_rate_limit(
245250
self._key_pool, current_key, response.headers,
246-
attempt, MAX_TRANSLATION_ATTEMPTS,
251+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
247252
)
248253
continue
249254

@@ -273,7 +278,8 @@ async def generate(
273278

274279
except httpx.TimeoutException as e:
275280
print(f"DeepSeek API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
276-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
281+
attempt += 1
282+
if attempt < MAX_TRANSLATION_ATTEMPTS:
277283
await asyncio.sleep(2)
278284
continue
279285
return None
@@ -309,21 +315,24 @@ async def generate(
309315
if not is_retryable_http_status(e.response.status_code):
310316
return None
311317

312-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
318+
attempt += 1
319+
if attempt < MAX_TRANSLATION_ATTEMPTS:
313320
await asyncio.sleep(2)
314321
continue
315322
return None
316323

317324
except json.JSONDecodeError as e:
318325
print(f"DeepSeek API JSON Decode Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
319-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
326+
attempt += 1
327+
if attempt < MAX_TRANSLATION_ATTEMPTS:
320328
await asyncio.sleep(2)
321329
continue
322330
return None
323331

324332
except Exception as e:
325333
print(f"DeepSeek API Unknown Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
326-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
334+
attempt += 1
335+
if attempt < MAX_TRANSLATION_ATTEMPTS:
327336
await asyncio.sleep(2)
328337
continue
329338
return None

‎src/core/llm/providers/gemini.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,11 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
190190
}
191191

192192
client = await self._get_client()
193-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
193+
# 429s have their own budget (rate_limit_events): rotating to a spare
194+
# key must not consume a transient-retry attempt (issue #217).
195+
attempt = 0
196+
rate_limit_events = 0
197+
while attempt < MAX_TRANSLATION_ATTEMPTS:
194198
current_key = await self._key_pool.acquire()
195199
headers = {
196200
"Content-Type": "application/json",
@@ -259,7 +263,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
259263

260264
except httpx.TimeoutException as e:
261265
print(f"Gemini API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
262-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
266+
attempt += 1
267+
if attempt < MAX_TRANSLATION_ATTEMPTS:
263268
await asyncio.sleep(2)
264269
continue
265270
return None
@@ -272,9 +277,10 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
272277

273278
# Handle rate limiting (429) — rotate key or sleep, raise if exhausted
274279
if e.response.status_code == 429:
280+
rate_limit_events += 1
275281
await handle_rate_limit(
276282
self._key_pool, current_key, e.response.headers,
277-
attempt, MAX_TRANSLATION_ATTEMPTS,
283+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
278284
)
279285
continue
280286

@@ -293,13 +299,15 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
293299
if not is_retryable_http_status(e.response.status_code):
294300
return None
295301

296-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
302+
attempt += 1
303+
if attempt < MAX_TRANSLATION_ATTEMPTS:
297304
await asyncio.sleep(2)
298305
continue
299306
return None
300307
except Exception as e:
301308
print(f"Gemini API Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
302-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
309+
attempt += 1
310+
if attempt < MAX_TRANSLATION_ATTEMPTS:
303311
await asyncio.sleep(2)
304312
continue
305313
return None

‎src/core/llm/providers/mistral.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,11 @@ async def generate(
215215
}
216216

217217
client = await self._get_client()
218-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
218+
# 429s have their own budget (rate_limit_events): rotating to a spare
219+
# key must not consume a transient-retry attempt (issue #217).
220+
attempt = 0
221+
rate_limit_events = 0
222+
while attempt < MAX_TRANSLATION_ATTEMPTS:
219223
current_key = await self._key_pool.acquire()
220224
headers = {
221225
"Authorization": f"Bearer {current_key}",
@@ -235,9 +239,10 @@ async def generate(
235239
raise ValueError("Invalid Mistral API key")
236240

237241
if response.status_code == 429:
242+
rate_limit_events += 1
238243
await handle_rate_limit(
239244
self._key_pool, current_key, response.headers,
240-
attempt, MAX_TRANSLATION_ATTEMPTS,
245+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
241246
)
242247
continue
243248

@@ -268,7 +273,8 @@ async def generate(
268273

269274
except httpx.TimeoutException as e:
270275
print(f"Mistral API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
271-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
276+
attempt += 1
277+
if attempt < MAX_TRANSLATION_ATTEMPTS:
272278
await asyncio.sleep(2)
273279
continue
274280
return None
@@ -306,21 +312,24 @@ async def generate(
306312
if not is_retryable_http_status(e.response.status_code):
307313
return None
308314

309-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
315+
attempt += 1
316+
if attempt < MAX_TRANSLATION_ATTEMPTS:
310317
await asyncio.sleep(2)
311318
continue
312319
return None
313320

314321
except json.JSONDecodeError as e:
315322
print(f"Mistral API JSON Decode Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
316-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
323+
attempt += 1
324+
if attempt < MAX_TRANSLATION_ATTEMPTS:
317325
await asyncio.sleep(2)
318326
continue
319327
return None
320328

321329
except Exception as e:
322330
print(f"Mistral API Unknown Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
323-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
331+
attempt += 1
332+
if attempt < MAX_TRANSLATION_ATTEMPTS:
324333
await asyncio.sleep(2)
325334
continue
326335
return None

‎src/core/llm/providers/openai.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,11 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
111111
payload["chat_template_kwargs"] = {"enable_thinking": False}
112112

113113
client = await self._get_client()
114-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
114+
# 429s have their own budget (rate_limit_events): rotating to a spare
115+
# key must not consume a transient-retry attempt (issue #217).
116+
attempt = 0
117+
rate_limit_events = 0
118+
while attempt < MAX_TRANSLATION_ATTEMPTS:
115119
current_key = await self._key_pool.acquire() if self._key_pool else None
116120
headers = {"Content-Type": "application/json"}
117121
if current_key:
@@ -166,7 +170,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
166170
else:
167171
print(f"{YELLOW}⚠️ OpenAI-compatible API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}{RESET}")
168172

169-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
173+
attempt += 1
174+
if attempt < MAX_TRANSLATION_ATTEMPTS:
170175
if self.log_callback:
171176
self.log_callback("llm_retry", f" Retrying in 2 seconds...")
172177
await asyncio.sleep(2)
@@ -210,9 +215,10 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
210215
# Handle rate limiting (429) — rotate key or sleep, raise if exhausted
211216
if (hasattr(e, 'response') and e.response is not None
212217
and e.response.status_code == 429 and self._key_pool):
218+
rate_limit_events += 1
213219
await handle_rate_limit(
214220
self._key_pool, current_key, e.response.headers,
215-
attempt, MAX_TRANSLATION_ATTEMPTS, self.log_callback,
221+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS, self.log_callback,
216222
)
217223
continue
218224

@@ -267,7 +273,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
267273
and not is_retryable_http_status(e.response.status_code)):
268274
return None
269275

270-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
276+
attempt += 1
277+
if attempt < MAX_TRANSLATION_ATTEMPTS:
271278
if self.log_callback:
272279
self.log_callback("llm_retry", f" Retrying in 2 seconds...")
273280
await asyncio.sleep(2)
@@ -303,7 +310,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
303310
else:
304311
print(f"{YELLOW}⚠️ OpenAI-compatible API JSON Decode Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}{RESET}")
305312

306-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
313+
attempt += 1
314+
if attempt < MAX_TRANSLATION_ATTEMPTS:
307315
if self.log_callback:
308316
self.log_callback("llm_retry", f" Retrying in 2 seconds...")
309317
await asyncio.sleep(2)
@@ -333,7 +341,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
333341
else:
334342
print(f"{YELLOW}⚠️ OpenAI-compatible API Unknown Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {type(e).__name__}: {e}{RESET}")
335343

336-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
344+
attempt += 1
345+
if attempt < MAX_TRANSLATION_ATTEMPTS:
337346
if self.log_callback:
338347
self.log_callback("llm_retry", f" Retrying in 2 seconds...")
339348
await asyncio.sleep(2)

‎src/core/llm/providers/openrouter.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,11 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
237237
}
238238

239239
client = await self._get_client()
240-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
240+
# 429s have their own budget (rate_limit_events): rotating to a spare
241+
# key must not consume a transient-retry attempt (issue #217).
242+
attempt = 0
243+
rate_limit_events = 0
244+
while attempt < MAX_TRANSLATION_ATTEMPTS:
241245
current_key = await self._key_pool.acquire()
242246
headers = {
243247
"Authorization": f"Bearer {current_key}",
@@ -311,7 +315,8 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
311315

312316
except httpx.TimeoutException as e:
313317
print(f"OpenRouter API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
314-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
318+
attempt += 1
319+
if attempt < MAX_TRANSLATION_ATTEMPTS:
315320
await asyncio.sleep(2)
316321
continue
317322
return None
@@ -323,9 +328,10 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
323328
error_message = f"{e} - {error_body}"
324329

325330
if e.response.status_code == 429:
331+
rate_limit_events += 1
326332
await handle_rate_limit(
327333
self._key_pool, current_key, e.response.headers,
328-
attempt, MAX_TRANSLATION_ATTEMPTS,
334+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
329335
)
330336
continue
331337

@@ -352,19 +358,22 @@ async def generate(self, prompt: str, timeout: int = REQUEST_TIMEOUT,
352358
if not is_retryable_http_status(e.response.status_code):
353359
return None
354360

355-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
361+
attempt += 1
362+
if attempt < MAX_TRANSLATION_ATTEMPTS:
356363
await asyncio.sleep(2)
357364
continue
358365
return None
359366
except json.JSONDecodeError as e:
360367
print(f"OpenRouter API JSON Decode Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
361-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
368+
attempt += 1
369+
if attempt < MAX_TRANSLATION_ATTEMPTS:
362370
await asyncio.sleep(2)
363371
continue
364372
return None
365373
except Exception as e:
366374
print(f"OpenRouter API Unknown Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
367-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
375+
attempt += 1
376+
if attempt < MAX_TRANSLATION_ATTEMPTS:
368377
await asyncio.sleep(2)
369378
continue
370379
return None

‎src/core/llm/providers/poe.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,11 @@ async def generate(
346346
}
347347

348348
client = await self._get_client()
349-
for attempt in range(MAX_TRANSLATION_ATTEMPTS):
349+
# 429s have their own budget (rate_limit_events): rotating to a spare
350+
# key must not consume a transient-retry attempt (issue #217).
351+
attempt = 0
352+
rate_limit_events = 0
353+
while attempt < MAX_TRANSLATION_ATTEMPTS:
350354
current_key = await self._key_pool.acquire()
351355
headers = {
352356
"Authorization": f"Bearer {current_key}",
@@ -373,9 +377,10 @@ async def generate(
373377
return None
374378

375379
if response.status_code == 429:
380+
rate_limit_events += 1
376381
await handle_rate_limit(
377382
self._key_pool, current_key, response.headers,
378-
attempt, MAX_TRANSLATION_ATTEMPTS,
383+
rate_limit_events, MAX_TRANSLATION_ATTEMPTS,
379384
)
380385
continue
381386

@@ -422,7 +427,8 @@ async def generate(
422427

423428
except httpx.TimeoutException as e:
424429
print(f"Poe API Timeout (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
425-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
430+
attempt += 1
431+
if attempt < MAX_TRANSLATION_ATTEMPTS:
426432
await asyncio.sleep(2)
427433
continue
428434
return None
@@ -456,21 +462,24 @@ async def generate(
456462
if not is_retryable_http_status(e.response.status_code):
457463
return None
458464

459-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
465+
attempt += 1
466+
if attempt < MAX_TRANSLATION_ATTEMPTS:
460467
await asyncio.sleep(2)
461468
continue
462469
return None
463470

464471
except json.JSONDecodeError as e:
465472
print(f"Poe API JSON Decode Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
466-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
473+
attempt += 1
474+
if attempt < MAX_TRANSLATION_ATTEMPTS:
467475
await asyncio.sleep(2)
468476
continue
469477
return None
470478

471479
except Exception as e:
472480
print(f"Poe API Unknown Error (attempt {attempt + 1}/{MAX_TRANSLATION_ATTEMPTS}): {e}")
473-
if attempt < MAX_TRANSLATION_ATTEMPTS - 1:
481+
attempt += 1
482+
if attempt < MAX_TRANSLATION_ATTEMPTS:
474483
await asyncio.sleep(2)
475484
continue
476485
return None

0 commit comments

Comments
 (0)