Skip to content

Commit add82f6

Browse files
committed
fix: round-5 bug-hunt sweep (cost gates, MCP stability, frontend a11y)
LLM cost-safety gates added to seven previously uncovered call sites that could otherwise drive unbounded spend on long inputs: experts/citation_validator, gap_discovery, conflict_resolver, curriculum, map_reduce (map + reduce), multi_pass (extract/cross-ref/synthesise), synthesis (synthesize + extract), task_planner (decompose + synth), embedding_cache (per-doc + query). MCP / skills: - executor MCPClientProxy now drains stderr so subprocess buffer can't fill and deadlock on chatty servers. - skills/definition trigger regex protected from ReDoS by capping pattern length and rejecting nested-quantifier backtracking risks. - mcp/client/pool emits a terminal ProgressEvent on tool completion (was stored but never triggered). Storage: - findings_store guards in-memory index mutations with threading.RLock. CLI: - research.py cancel merges nested asyncio.run() into a single loop so lookup + cancel share the same event loop. Frontend: - a11y: htmlFor/id pairing for settings + research-studio inputs, aria-label on search/slider widgets, aria-valuemin/max/now on cost slider. - responsive: trace-explorer decision sidebar hidden on <lg, expert-profile conversation sidebar hidden on <md. - dead code removal: use-local-storage, use-media-query, activity-feed, memory-indicator.
1 parent 14df7bf commit add82f6

26 files changed

Lines changed: 709 additions & 240 deletions

deepr/cli/commands/research.py

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -515,25 +515,7 @@ async def cancel_job():
515515
return job, "Job already failed"
516516
return job, None
517517

518-
job, error = asyncio.run(cancel_job())
519-
if error:
520-
print_error(error)
521-
raise click.Abort()
522-
523-
console.print("\nJob Details:")
524-
console.print(f" ID: {job.id}")
525-
console.print(f" Status: {job.status.name}")
526-
console.print(f" Prompt: {job.prompt[:80]}{'...' if len(job.prompt) > 80 else ''}")
527-
if job.provider_job_id:
528-
console.print(f" Provider Job ID: {job.provider_job_id}")
529-
530-
if not yes and not click.confirm("\nCancel this job?"):
531-
print_warning("Cancelled")
532-
return
533-
534-
print_success("Cancelling job...")
535-
536-
async def do_cancel():
518+
async def do_cancel(job):
537519
if job.provider_job_id:
538520
try:
539521
await provider.cancel_job(job.provider_job_id)
@@ -542,10 +524,38 @@ async def do_cancel():
542524
console.print(f" Warning: Could not cancel at provider: {e}")
543525
await queue.update_status(job_id=job.id, status=JobStatus.FAILED, error="Cancelled by user")
544526

545-
import asyncio as _asyncio
527+
# Single event loop spans the lookup AND the cancel so the
528+
# sqlite queue / aiohttp provider connections opened during
529+
# lookup stay valid for the cancel call. The previous
530+
# ``asyncio.run`` per step closed the loop between them,
531+
# binding any reused stateful objects to a now-closed loop
532+
# and intermittently raising "got Future attached to a
533+
# different loop".
534+
loop = asyncio.new_event_loop()
535+
try:
536+
asyncio.set_event_loop(loop)
537+
job, error = loop.run_until_complete(cancel_job())
538+
if error:
539+
print_error(error)
540+
raise click.Abort()
541+
542+
console.print("\nJob Details:")
543+
console.print(f" ID: {job.id}")
544+
console.print(f" Status: {job.status.name}")
545+
console.print(f" Prompt: {job.prompt[:80]}{'...' if len(job.prompt) > 80 else ''}")
546+
if job.provider_job_id:
547+
console.print(f" Provider Job ID: {job.provider_job_id}")
548+
549+
if not yes and not click.confirm("\nCancel this job?"):
550+
print_warning("Cancelled")
551+
return
546552

547-
_asyncio.run(do_cancel())
548-
print_success("Job cancelled successfully!")
553+
print_success("Cancelling job...")
554+
loop.run_until_complete(do_cancel(job))
555+
print_success("Job cancelled successfully!")
556+
finally:
557+
loop.close()
558+
asyncio.set_event_loop(None)
549559

550560
except Exception as e:
551561
print_error(f"Error: {e}")

deepr/experts/citation_validator.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,35 @@ async def _validate_batch(
102102
prompt_parts.append("Output ONLY the JSON array, no other text.")
103103
prompt = "\n".join(prompt_parts)
104104

105+
# Pre-flight cost-safety gate. Citation validation can fire many
106+
# paid LLM calls in a long-running expert workflow; without this
107+
# gate the daily / monthly budget is bypassed.
108+
try:
109+
from deepr.experts.cost_safety import get_cost_safety_manager
110+
111+
cost_safety = get_cost_safety_manager()
112+
est_cost = 0.02 # gpt-5.2 batch validation; low reasoning effort
113+
allowed, deny_reason, _ = cost_safety.check_operation(
114+
session_id="citation_validator",
115+
operation_type="citation_validation",
116+
estimated_cost=est_cost,
117+
require_confirmation=False,
118+
)
119+
if not allowed:
120+
logger.warning("Citation validation blocked by cost-safety: %s", deny_reason)
121+
return [
122+
SourceValidation(
123+
source_id=src.id,
124+
claim_id=claim.id,
125+
support_class=SupportClass.UNCERTAIN,
126+
explanation=f"Skipped: {deny_reason}",
127+
)
128+
for claim, src, _ in pairs
129+
]
130+
except Exception: # never let cost-safety bookkeeping mask the result
131+
cost_safety = None # type: ignore[assignment]
132+
est_cost = 0.0
133+
105134
try:
106135
client = await self._get_client()
107136
response = await client.chat.completions.create(
@@ -142,6 +171,26 @@ async def _validate_batch(
142171
explanation=result.get("explanation", ""),
143172
)
144173
)
174+
175+
# Settle the cost into the canonical ledger.
176+
if cost_safety is not None:
177+
try:
178+
actual_cost = est_cost
179+
if response.usage:
180+
from deepr.experts.chat import _chat_token_cost as _tc
181+
182+
actual_cost = _tc(response.usage, self.model)
183+
cost_safety.record_cost(
184+
session_id="citation_validator",
185+
operation_type="citation_validation",
186+
actual_cost=float(actual_cost),
187+
provider="openai",
188+
model=self.model,
189+
source="experts.citation_validator.validate_claims",
190+
)
191+
except Exception:
192+
pass
193+
145194
return validations
146195

147196
except Exception as e:

deepr/experts/conflict_resolver.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,25 @@ async def _llm_detect_contradictions(self, pairs: list[tuple[Belief, Belief]]) -
141141

142142
prompt_parts.append("\nOutput ONLY a JSON array like [0, 3, 5] or [] if none contradict.")
143143

144+
# Cost-safety gate on the LLM-driven contradiction scan.
145+
try:
146+
from deepr.experts.cost_safety import get_cost_safety_manager
147+
148+
cost_safety = get_cost_safety_manager()
149+
est_cost = 0.02
150+
allowed, deny_reason, _ = cost_safety.check_operation(
151+
session_id="conflict_resolver",
152+
operation_type="conflict_detection",
153+
estimated_cost=est_cost,
154+
require_confirmation=False,
155+
)
156+
if not allowed:
157+
logger.warning("Conflict detection blocked by cost-safety: %s", deny_reason)
158+
return []
159+
except Exception:
160+
cost_safety = None # type: ignore[assignment]
161+
est_cost = 0.0
162+
144163
try:
145164
client = await self._get_client()
146165
response = await client.chat.completions.create(
@@ -155,6 +174,21 @@ async def _llm_detect_contradictions(self, pairs: list[tuple[Belief, Belief]]) -
155174
if text.startswith("```"):
156175
text = text.split("\n", 1)[-1].rsplit("```", 1)[0]
157176
indices = json.loads(text)
177+
if cost_safety is not None:
178+
try:
179+
from deepr.experts.chat import _chat_token_cost as _tc
180+
181+
actual_cost = _tc(response.usage, "gpt-5.2") if response.usage else est_cost
182+
cost_safety.record_cost(
183+
session_id="conflict_resolver",
184+
operation_type="conflict_detection",
185+
actual_cost=float(actual_cost),
186+
provider="openai",
187+
model="gpt-5.2",
188+
source="experts.conflict_resolver.detect_pairs",
189+
)
190+
except Exception:
191+
pass
158192
return [pairs[i] for i in indices if 0 <= i < len(pairs)]
159193
except Exception as e:
160194
logger.warning("LLM contradiction detection failed: %s", e)

deepr/experts/curriculum.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,27 @@ async def _call_gpt5_with_retry(
440440
# Use the cheapest curriculum model (sorted by cost)
441441
model_name = curriculum_models[0].model if curriculum_models else "gpt-5.2"
442442

443+
# Cost-safety gate before the curriculum LLM call.
444+
try:
445+
from deepr.experts.cost_safety import get_cost_safety_manager
446+
447+
_cost_safety = get_cost_safety_manager()
448+
_est_cost = 0.05 # Curriculum prompts are larger; bumpier estimate
449+
_allowed, _deny_reason, _ = _cost_safety.check_operation(
450+
session_id="curriculum",
451+
operation_type="curriculum_plan",
452+
estimated_cost=_est_cost,
453+
require_confirmation=False,
454+
)
455+
if not _allowed:
456+
logger.warning("Curriculum planning blocked by cost-safety: %s", _deny_reason)
457+
raise RuntimeError(f"Curriculum blocked: {_deny_reason}")
458+
except RuntimeError:
459+
raise
460+
except Exception:
461+
_cost_safety = None # type: ignore[assignment]
462+
_est_cost = 0.0
463+
443464
response_obj = await client.chat.completions.create(
444465
model=model_name,
445466
messages=[
@@ -456,6 +477,23 @@ async def _call_gpt5_with_retry(
456477
# Extract response from chat completion
457478
response = response_obj.choices[0].message.content or ""
458479

480+
# Settle the cost into the canonical ledger.
481+
if _cost_safety is not None:
482+
try:
483+
from deepr.experts.chat import _chat_token_cost as _tc
484+
485+
_actual_cost = _tc(response_obj.usage, model_name) if response_obj.usage else _est_cost
486+
_cost_safety.record_cost(
487+
session_id="curriculum",
488+
operation_type="curriculum_plan",
489+
actual_cost=float(_actual_cost),
490+
provider="openai",
491+
model=model_name,
492+
source="experts.curriculum.generate_curriculum",
493+
)
494+
except Exception:
495+
pass
496+
459497
if progress:
460498
progress.complete("Done")
461499

deepr/experts/embedding_cache.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,16 @@ async def add_documents(
135135
if not uncached:
136136
return 0
137137

138+
# Cost-safety gate. Embedding a corpus is cheap-per-doc but can
139+
# add up over thousands of files; the rate limit also exposes a
140+
# real-money path through misconfigured loops. Gate per-document.
141+
try:
142+
from deepr.experts.cost_safety import get_cost_safety_manager
143+
144+
_cost_safety = get_cost_safety_manager()
145+
except Exception:
146+
_cost_safety = None # type: ignore[assignment]
147+
138148
# Batch embed new documents (more efficient than one-by-one)
139149
new_embeddings = []
140150
new_metadata = []
@@ -146,9 +156,38 @@ async def add_documents(
146156
# Truncate content for embedding (model limit)
147157
embed_content = content[:8000]
148158

159+
if _cost_safety is not None:
160+
try:
161+
_est = 0.0002 # text-embedding-3-small ~$0.02/M tokens
162+
_allowed, _reason, _ = _cost_safety.check_operation(
163+
session_id=f"embed:{self.expert_name}",
164+
operation_type="embed_document",
165+
estimated_cost=_est,
166+
require_confirmation=False,
167+
)
168+
if not _allowed:
169+
logger.warning("Embedding for %s blocked by cost-safety: %s", filename, _reason)
170+
continue
171+
except Exception:
172+
_est = 0.0
173+
else:
174+
_est = 0.0
175+
149176
try:
150177
response = await client.embeddings.create(model=model, input=embed_content)
151178
embedding = np.array(response.data[0].embedding)
179+
if _cost_safety is not None:
180+
try:
181+
_cost_safety.record_cost(
182+
session_id=f"embed:{self.expert_name}",
183+
operation_type="embed_document",
184+
actual_cost=float(_est),
185+
provider="openai",
186+
model=model,
187+
source="experts.embedding_cache.add_documents",
188+
)
189+
except Exception:
190+
pass
152191

153192
content_hash = self._content_hash(content)
154193

@@ -205,10 +244,41 @@ async def search(self, query: str, client, top_k: int = 5, model: str = "text-em
205244
if self.embeddings is None or len(self.embeddings) == 0:
206245
return []
207246

247+
# Cost-safety gate for query embedding.
248+
try:
249+
from deepr.experts.cost_safety import get_cost_safety_manager
250+
251+
_cost_safety = get_cost_safety_manager()
252+
_est = 0.0001
253+
_allowed, _reason, _ = _cost_safety.check_operation(
254+
session_id=f"embed_query:{self.expert_name}",
255+
operation_type="embed_query",
256+
estimated_cost=_est,
257+
require_confirmation=False,
258+
)
259+
if not _allowed:
260+
logger.warning("Query embedding blocked by cost-safety: %s", _reason)
261+
return []
262+
except Exception:
263+
_cost_safety = None # type: ignore[assignment]
264+
_est = 0.0
265+
208266
# Embed query (single API call)
209267
try:
210268
response = await client.embeddings.create(model=model, input=query)
211269
query_embedding = np.array(response.data[0].embedding)
270+
if _cost_safety is not None:
271+
try:
272+
_cost_safety.record_cost(
273+
session_id=f"embed_query:{self.expert_name}",
274+
operation_type="embed_query",
275+
actual_cost=float(_est),
276+
provider="openai",
277+
model=model,
278+
source="experts.embedding_cache.search",
279+
)
280+
except Exception:
281+
pass
212282
except Exception as e:
213283
logger.error("Error embedding query: %s", e)
214284
return []

deepr/experts/gap_discovery.py

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,26 @@ async def _generate_gaps_for_thin_areas(
204204
"Priority 1-5 (5=most important). Output ONLY the JSON."
205205
)
206206

207+
# Pre-flight cost-safety gate so the gap-discovery pipeline can't
208+
# silently fan out paid LLM calls without daily-budget enforcement.
209+
try:
210+
from deepr.experts.cost_safety import get_cost_safety_manager
211+
212+
cost_safety = get_cost_safety_manager()
213+
est_cost = 0.02
214+
allowed, deny_reason, _ = cost_safety.check_operation(
215+
session_id="gap_discovery",
216+
operation_type="gap_discovery_thin_areas",
217+
estimated_cost=est_cost,
218+
require_confirmation=False,
219+
)
220+
if not allowed:
221+
logger.warning("Gap discovery (thin areas) blocked by cost-safety: %s", deny_reason)
222+
return []
223+
except Exception:
224+
cost_safety = None # type: ignore[assignment]
225+
est_cost = 0.0
226+
207227
try:
208228
client = await self._get_client()
209229
response = await client.chat.completions.create(
@@ -217,7 +237,23 @@ async def _generate_gaps_for_thin_areas(
217237
text = response.choices[0].message.content or "[]"
218238
if text.startswith("```"):
219239
text = text.split("\n", 1)[-1].rsplit("```", 1)[0]
220-
return json.loads(text)
240+
result = json.loads(text)
241+
if cost_safety is not None:
242+
try:
243+
from deepr.experts.chat import _chat_token_cost as _tc
244+
245+
actual_cost = _tc(response.usage, "gpt-5.2") if response.usage else est_cost
246+
cost_safety.record_cost(
247+
session_id="gap_discovery",
248+
operation_type="gap_discovery_thin_areas",
249+
actual_cost=float(actual_cost),
250+
provider="openai",
251+
model="gpt-5.2",
252+
source="experts.gap_discovery._generate_gaps_for_thin_areas",
253+
)
254+
except Exception:
255+
pass
256+
return result
221257
except Exception as e:
222258
logger.warning("Gap generation failed: %s", e)
223259
return []

0 commit comments

Comments
 (0)