fix(recon): catch inner crt.sh timeout in full_domain_report closures (v1.33.20)

The two helper closures that share the crt.sh task wrapped their inner
asyncio.wait_for(asyncio.shield(...)) without a guard, so a slow upstream
made the inner timeout raise inside the closure; the task ended with an
unretrieved exception and the outer await failed the whole domain report.

Catch the inner TimeoutError in both closures and fall through to the
partial-success path (empty CT data + crt_sh_timeout sentinel). The
certificates branch already accepted that input via check_ct_logs; extend
enumerate_subdomains / _crtsh_subdomains with an optional crtsh_error so
the subdomains branch can honestly surface crtsh_status='timeout' instead
of masquerading as 'ok' with an empty result. Defaults preserve existing
call sites — no schema delta, no breaking change.

Add a regression test that hangs the crt.sh fetch and asserts both
branches report the timeout via the documented status fields.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: UPinar

app/config.py

@@ -12,7 +12,7 @@
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-VERSION = "1.33.19"
+VERSION = "1.33.20"
 MCP_TOOL_COUNT = 53  # v1.33.0: +tech_stack_cve_audit (MCP-only composite)
 MCP_RESOURCE_COUNT = 7  # v1.23.0: atlas+d3fend+cwe (4 templates + 3 catalogs)
 MCP_PROMPT_COUNT = 3  # v1.23.0: security_audit, vulnerability_check, contrast_triage

app/domain/recon.py

@@ -334,8 +334,14 @@ def _parse_whois(text: str) -> dict:
 # === Subdomains ===
 
 
-async def enumerate_subdomains(domain: str, crtsh_data: list | None = None) -> dict:
-    """Enumerate subdomains via DNS brute force + crt.sh CT logs."""
+async def enumerate_subdomains(domain: str, crtsh_data: list | None = None, crtsh_error: str | None = None) -> dict:
+    """Enumerate subdomains via DNS brute force + crt.sh CT logs.
+
+    When the caller pre-fetched crt.sh and surfaced an upstream failure (e.g.,
+    `crt_sh_timeout` from full_domain_report's shared f_crtsh task), pass that
+    string as `crtsh_error` so the response honestly reports `crtsh_status`
+    instead of masquerading as "ok" with an empty result.
+    """
     found_wordlist: set[str] = set()
     warnings: list[str] = []
 
@@ -354,7 +360,7 @@ def _resolve_sub(sub):
         if result:
             found_wordlist.add(result)
 
-    found_crtsh, crtsh_warnings, crtsh_status = await _crtsh_subdomains(domain, crtsh_data)
+    found_crtsh, crtsh_warnings, crtsh_status = await _crtsh_subdomains(domain, crtsh_data, fetch_error=crtsh_error)
     warnings.extend(crtsh_warnings)
 
     all_found = sorted(found_wordlist | set(found_crtsh))
@@ -445,7 +451,9 @@ async def _fetch_crtsh(query: str) -> tuple[list, str | None]:
 }
 
 
-async def _crtsh_subdomains(domain: str, data: list | None = None) -> tuple[list, list, str]:
+async def _crtsh_subdomains(
+    domain: str, data: list | None = None, fetch_error: str | None = None
+) -> tuple[list, list, str]:
     """Extract subdomain names from crt.sh data.
 
     Returns:
@@ -454,10 +462,17 @@ async def _crtsh_subdomains(domain: str, data: list | None = None) -> tuple[list
         callers distinguish "CT lookup confirmed empty" from "CT lookup failed";
         the legacy (subs, warnings) shape conflated the two and downstream tools
         could not tell the difference between an actually-tiny domain and a
-        crt.sh outage.
+        crt.sh outage. The optional `fetch_error` argument lets the caller
+        propagate an upstream failure (e.g., crt_sh_timeout) when it pre-fetched
+        the data itself — mirrors check_ct_logs's `crtsh_error` parameter so
+        both halves of the recon report agree on whether crt.sh delivered.
     """
     warnings: list[str] = []
     status = "ok"
+    if fetch_error:
+        warnings.append(fetch_error)
+        status = _CRTSH_STATUS_BY_ERROR.get(fetch_error, "error")
+        return ([], warnings, status)
     if data is None:
         data, fetch_error = await _fetch_crtsh(f"%.{domain}")
         if fetch_error:
@@ -1446,11 +1461,21 @@ async def full_domain_report(
         async def _subs_with_crtsh():
             # asyncio.shield prevents cancellation of f_crtsh from propagating
             # back into the shared crtsh task — both _subs and _ct depend on it.
-            data, _ = await asyncio.wait_for(asyncio.shield(f_crtsh), timeout=CRTSH_TIMEOUT + 2)
-            return await enumerate_subdomains(domain, crtsh_data=data)
+            # Inner TimeoutError must be caught here so this closure's task
+            # completes cleanly with partial result; otherwise the unretrieved
+            # exception triggers 'Task exception was never retrieved' (S253 #1).
+            crtsh_err: str | None = None
+            try:
+                data, crtsh_err = await asyncio.wait_for(asyncio.shield(f_crtsh), timeout=CRTSH_TIMEOUT + 2)
+            except asyncio.TimeoutError:
+                data, crtsh_err = [], "crt_sh_timeout"
+            return await enumerate_subdomains(domain, crtsh_data=data, crtsh_error=crtsh_err)
 
         async def _ct_with_crtsh():
-            data, fetch_error = await asyncio.wait_for(asyncio.shield(f_crtsh), timeout=CRTSH_TIMEOUT + 2)
+            try:
+                data, fetch_error = await asyncio.wait_for(asyncio.shield(f_crtsh), timeout=CRTSH_TIMEOUT + 2)
+            except asyncio.TimeoutError:
+                data, fetch_error = [], "crt_sh_timeout"
             return await check_ct_logs(domain, data, crtsh_error=fetch_error)
 
         f_subs = asyncio.create_task(_subs_with_crtsh())

app/tests/test_domain.py

@@ -3445,6 +3445,53 @@ async def _run():
         )
 
 
+class TestCtCrtshInnerTimeoutOrphan:
+    """Regression for S253 #1 (113/24h prod orphan logs). When _fetch_crtsh
+    is slow enough that the INNER asyncio.wait_for(asyncio.shield(f_crtsh),
+    timeout=CRTSH_TIMEOUT+2) times out FIRST, the TimeoutError must be
+    caught INSIDE _ct_with_crtsh() and _subs_with_crtsh() so f_certs/f_subs
+    complete cleanly with partial results. Pre-fix: TimeoutError propagates
+    to f_certs task, outer wait_for at line ~1488 fails the whole report,
+    and asyncio logs 'Task exception was never retrieved'."""
+
+    def test_inner_crtsh_timeout_returns_partial_certificates(self, monkeypatch):
+        async def _hang_crtsh(_q):
+            await asyncio.Event().wait()
+            return ([], None)
+
+        async def _ok_threat(*_a, **_kw):
+            return {
+                "urlhaus_status": "ok",
+                "url_count": 0,
+                "urls_online": 0,
+                "threat_types": [],
+                "tags": [],
+                "urls": [],
+            }
+
+        async def _ok_headers(*_a, **_kw):
+            return {"headers": {}, "status": 200}
+
+        monkeypatch.setattr("domain.recon.CRTSH_TIMEOUT", 0)
+        monkeypatch.setattr("domain.recon.dns_lookup", lambda d: {"a": [], "txt": [], "mx": [], "ns": []})
+        monkeypatch.setattr("domain.recon.reverse_dns", lambda d: {"ip": None})
+        monkeypatch.setattr("domain.recon.ssl_info", lambda d, ip: {})
+        monkeypatch.setattr("domain.recon.whois_lookup", lambda d: {})
+        monkeypatch.setattr("domain.recon.email_security", lambda d, txt: {"grade": "F"})
+        monkeypatch.setattr("domain.recon._fetch_crtsh", _hang_crtsh)
+        monkeypatch.setattr("domain.recon.fetch_live_headers", _ok_headers)
+        monkeypatch.setattr("domain.threat.check_urlhaus", _ok_threat)
+
+        from domain.recon import full_domain_report
+
+        result = asyncio.run(full_domain_report("example.com"))
+
+        assert result["certificates"]["error"] == "crt_sh_timeout"
+        assert result["certificates"]["crtsh_status"] == "timeout"
+        assert result["certificates"]["total_certificates"] == 0
+        assert result["subdomains"]["crtsh_status"] == "timeout"
+
+
 # =========== fetch_live_page connection-release (cancel-without-await leak) ===========