diff --git a/examples/parallel-decisions.md b/examples/parallel-decisions.md new file mode 100644 index 000000000..340da6753 --- /dev/null +++ b/examples/parallel-decisions.md @@ -0,0 +1,64 @@ +# Parallel Decisions through treg + +Connect your team's Parallel API key once. Calls with this key are billed by Parallel to your +Parallel account and do not debit your treg balance. + +```bash +treg secret add parallel --value "$PARALLEL_API_KEY" +treg catalog get parallel.ai-judge.decide +``` + +One predicate question: + +```bash +treg call parallel.ai-judge.decide --data '{ + "state": {"company": "Parallel", "website": "https://parallel.ai"}, + "questions": { + "has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"} + }, + "include_excerpts": false +}' +``` + +Several questions can share the same state and web evidence in one request: + +```bash +treg call parallel.ai-judge.decide --data '{ + "state": {"company": "Parallel", "website": "https://parallel.ai"}, + "questions": { + "has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"}, + "product_type": { + "type": "choice", + "instructions": "Which description best fits Parallel?", + "criteria": {"web_api": "Offers a web API", "desktop_app": "Offers a desktop app"} + }, + "documentation_level": { + "type": "score", + "instructions": "How detailed are Parallel's public API docs?", + "criteria": ["No docs", "Partial docs", "Detailed docs"] + } + } +}' +``` + +Read each result under `answers.`. The response includes the vendor's `decision_id`, +`mode`, token `usage`, and shared `citations`. A question can return `type: "refusal"` instead of a +decision value. See [Parallel's question types](https://docs.parallel.ai/decisions/question-types) +for criteria and limits. + +## Verification ledger + +Checked against Parallel's [API reference](https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions), +[question types](https://docs.parallel.ai/decisions/question-types), +[best practices](https://docs.parallel.ai/decisions/best-practices), +[pricing](https://docs.parallel.ai/getting-started/pricing), and +[rate limits](https://docs.parallel.ai/getting-started/rate-limits) on 2026-10-30. +The documented rate is $0.001 per `fast` request with up to 100 questions. + +`tests/test_parallel_decisions.py` uses a mocked upstream for single and multiple questions, +response fields and citations, upstream error statuses, own-key precedence, and the platform-key +gate. `src/treg/catalog/examples/parallel.ai-judge.decide.json` is synthetic and sanitized; it is +not a captured live response. No live Parallel Decisions call or vendor billing observation was +made. Platform-funded access remains disabled until a maintainer verifies the platform key can +call Decisions, confirms actual charges and error billing against the shared account, and selects +settlement behavior from that evidence. diff --git a/src/treg/catalog/examples/parallel.ai-judge.decide.json b/src/treg/catalog/examples/parallel.ai-judge.decide.json new file mode 100644 index 000000000..078f0bf82 --- /dev/null +++ b/src/treg/catalog/examples/parallel.ai-judge.decide.json @@ -0,0 +1,25 @@ +{ + "mode": "fast", + "answers": { + "has_api": {"type": "noul", "noul": 0.99}, + "product_type": { + "type": "choice", + "choice": "web_api", + "probabilities": {"web_api": 0.95, "desktop_app": 0.05}, + "confidence": 0.91 + }, + "documentation_level": { + "type": "score", + "score": 1.8, + "legend": {"0": "No public documentation", "1": "Partial documentation", "2": "Detailed documentation"}, + "probabilities": {"0": 0.0, "1": 0.2, "2": 0.8}, + "confidence": 0.86 + }, + "unanswerable": {"type": "refusal"} + }, + "usage": {"input_tokens": 1200, "output_tokens": 0}, + "citations": [ + {"url": "https://example.com/docs", "title": "Example documentation", "excerpts": ["Example excerpt."]} + ], + "decision_id": "decisions_00000000000000000000000000000000" +} diff --git a/src/treg/catalog/parallel.yaml b/src/treg/catalog/parallel.yaml index 23c9af457..7f5d2bab7 100644 --- a/src/treg/catalog/parallel.yaml +++ b/src/treg/catalog/parallel.yaml @@ -2,11 +2,44 @@ provider: parallel source: docs: https://docs.parallel.ai/getting-started/overview openapi: https://docs.parallel.ai/public-openapi.json - curated: '2026-10-09' + curated: '2026-10-30' pricing_url: https://docs.parallel.ai/getting-started/pricing -limits: "Search, Extract and Entity Search each allow 600 requests per minute on a rolling 60-second window. HTTP 429 is a rate limit; HTTP 402 is insufficient credit, which in-flight requests count against." +limits: "Search, Extract, Entity Search and Decisions each allow 600 requests per minute on a rolling 60-second window. HTTP 429 is a rate limit; HTTP 402 is insufficient credit, which in-flight requests count against." endpoints: + - id: parallel.ai-judge.decide + capability: ai-judge.decide + platform: ai-judge + domain: decisions + scope: any_account + method: POST + path: /v1/decisions + platform_blocked: "Platform-key billing and settlement for Decisions have not been live-verified; connect your team's own Parallel key." + name: Make web-grounded decisions + summary: Answer up to 100 named predicate, choice or score questions using shared state and current web evidence. + input: + bodyType: json + body: + state: {type: any, required: true, example: {company: "Parallel", website: "https://parallel.ai"}, note: "Any JSON value shared by all questions. State plus the largest question is limited to 32,000 estimated input tokens."} + questions: {type: object, required: true, example: {has_api: {type: noul, instructions: "Does Parallel offer an API?"}}, note: "1–100 names matching [A-Za-z0-9_-]+. Each value has type noul, choice or score and instructions. Noul may include criteria {true, false}; choice requires 2–255 named option descriptions; score requires 2–10 ordered level descriptions. Each rendered question is limited to 5,000 characters."} + mode: {type: string, required: false, default: fast, enum: [fast], note: "Only fast mode is currently supported."} + include_excerpts: {type: boolean, required: false, default: true, note: "False keeps citation URLs and titles but returns empty excerpts arrays."} + note: "Answers are keyed by question name. Noul returns a probability; choice returns the chosen key and option probabilities; score returns a weighted mean, legend and probabilities. Choice and score may include confidence. Any answer may be a refusal. The raw response also includes mode, decision_id, token usage and citations shared across all questions; citations are not mapped to individual answers." + test_request: {body: {state: {company: "Parallel", website: "https://parallel.ai"}, questions: {has_api: {type: noul, instructions: "Does Parallel offer an API?"}}, include_excerpts: false}} + cost: + type: per_call + value: 0.001 + currency: USD + unit: call + source: docs + source_url: https://docs.parallel.ai/getting-started/pricing + checked: '2026-10-30' + confidence: documented + note: "$0.001 per fast request, including up to 100 questions. Documented rate only; platform-key billing and settlement remain unverified, so this tool is team-key only. The response's usage is token counts, not a charge." + added: '2026-10-30' + example_response: examples/parallel.ai-judge.decide.json + docs_url: https://docs.parallel.ai/decisions/decisions-quickstart + - id: parallel.web.search capability: web.search platform: web diff --git a/tests/test_parallel_decisions.py b/tests/test_parallel_decisions.py new file mode 100644 index 000000000..144f24853 --- /dev/null +++ b/tests/test_parallel_decisions.py @@ -0,0 +1,165 @@ +"""Parallel Decisions catalog contract and faithful own-key relay (mocked upstream).""" + +import json +from pathlib import Path + +import httpx +import pytest + +from treg import api as A +from treg.config import get_settings +from treg.domain.catalog import store as catalog_store + + +ENDPOINT = "parallel.ai-judge.decide" +FIXTURE = (Path(__file__).resolve().parents[1] / "src/treg/catalog/examples" + / f"{ENDPOINT}.json") + + +class _JSONStream(httpx.AsyncByteStream): + def __init__(self, payload): + self.payload = json.dumps(payload).encode() + + async def __aiter__(self): + yield self.payload + + +def _response(status, payload, *, headers=None): + return httpx.Response(status, headers={"content-type": "application/json", **(headers or {})}, + stream=_JSONStream(payload)) + + +async def _balance(clients): + org_id = (await clients.get("/orgs")).json()[0]["org_id"] + return (await clients.get(f"/orgs/{org_id}/balance")).json()["balance_micro"] + + +def test_decisions_catalog_contract(): + catalog = catalog_store.load() + endpoint = catalog.by_id[ENDPOINT] + assert endpoint["capability"] == "ai-judge.decide" + assert (endpoint["method"], endpoint["path"]) == ("POST", "/v1/decisions") + assert endpoint["input"]["body"]["state"]["type"] == "any" + assert endpoint["input"]["body"]["questions"]["type"] == "object" + assert endpoint["input"]["body"]["include_excerpts"]["default"] is True + assert endpoint["cost"]["value"] == 0.001 + assert endpoint["cost"]["confidence"] == "documented" + assert not catalog.platform_eligible(endpoint) + response = json.loads(FIXTURE.read_text()) + assert {"noul", "choice", "score", "refusal"} == { + answer["type"] for answer in response["answers"].values() + } + + +async def test_decisions_is_discoverable(clients): + response = await clients.get(f"/catalog/endpoints/{ENDPOINT}") + assert response.status_code == 200 + assert response.json()["endpoint"]["id"] == ENDPOINT + assert response.json()["endpoint"]["platform_eligible"] is False + assert response.json()["example_response"]["decision_id"].startswith("decisions_") + + +@pytest.mark.parametrize("state", [None, "Parallel", {"company": "Parallel"}]) +async def test_decisions_single_question_preserves_json_state_and_response(clients, monkeypatch, state): + secret = await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"}) + assert secret.status_code == 200, secret.text + request_body = { + "state": state, + "questions": {"has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"}}, + "include_excerpts": False, + } + reply = json.loads(FIXTURE.read_text()) + seen = [] + + def serve(request): + seen.append((request.url.path, request.headers.get("x-api-key"), json.loads(request.content))) + return _response(200, reply) + + before = await _balance(clients) + async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream: + monkeypatch.setattr(A.app.state, "http", upstream) + response = await clients.post(f"/call/{ENDPOINT}", json=request_body) + assert response.status_code == 200, response.text + assert response.json() == reply + assert seen == [("/v1/decisions", "OWN-PARALLEL", request_body)] + assert "x-treg-cost-micro" not in response.headers + assert await _balance(clients) == before + + +async def test_decisions_multiple_question_types_and_citations_are_relayed(clients, monkeypatch): + await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"}) + body = { + "state": {"company": "Parallel", "website": "https://parallel.ai"}, + "mode": "fast", + "questions": { + "has_api": {"type": "noul", "instructions": "Does Parallel offer an API?", + "criteria": {"true": "There is an API.", "false": "There is no API."}}, + "product_type": {"type": "choice", "instructions": "What kind of product is it?", + "criteria": {"web_api": "A web API", "desktop_app": "A desktop app"}}, + "documentation_level": {"type": "score", "instructions": "How detailed are the docs?", + "criteria": ["None", "Partial", "Detailed"]}, + }, + } + reply = json.loads(FIXTURE.read_text()) + + def serve(request): + assert request.headers["x-api-key"] == "OWN-PARALLEL" + assert json.loads(request.content) == body + return _response(200, reply) + + async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream: + monkeypatch.setattr(A.app.state, "http", upstream) + response = await clients.post(f"/call/{ENDPOINT}", json=body) + assert response.status_code == 200, response.text + assert response.json() == reply + assert response.json()["answers"]["documentation_level"]["confidence"] == 0.86 + assert response.json()["citations"][0]["excerpts"] == ["Example excerpt."] + + +@pytest.mark.parametrize("status", [401, 422, 429, 503]) +async def test_decisions_upstream_errors_follow_the_relay(clients, monkeypatch, status): + await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"}) + error = {"type": "error", "error": {"ref_id": "example", "message": "Example error"}} + + def serve(request): + return _response(status, error, headers={"retry-after": "3"}) + + before = await _balance(clients) + async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream: + monkeypatch.setattr(A.app.state, "http", upstream) + response = await clients.post(f"/call/{ENDPOINT}", json={ + "state": {}, "questions": {"q": {"type": "noul", "instructions": "Is it true?"}}, + }) + assert response.status_code == status, response.text + assert response.json() == error + assert await _balance(clients) == before + + +async def test_decisions_never_uses_platform_key_until_billing_is_verified(clients, monkeypatch): + monkeypatch.setenv("TREG_PLATFORM_KEY_PARALLEL", "PLATFORM-PARALLEL") + monkeypatch.setenv("TREG_PLATFORM_PROVIDERS", "parallel") + get_settings.cache_clear() + seen = [] + body = {"state": {}, "questions": {"q": {"type": "noul", "instructions": "Is it true?"}}} + + def serve(request): + seen.append(request.headers.get("x-api-key")) + return _response(200, json.loads(FIXTURE.read_text())) + + try: + before = await _balance(clients) + async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream: + monkeypatch.setattr(A.app.state, "http", upstream) + missing = await clients.post(f"/call/{ENDPOINT}", json=body) + assert missing.status_code == 404 + assert seen == [] + assert await _balance(clients) == before + + await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"}) + own = await clients.post(f"/call/{ENDPOINT}", json=body) + assert own.status_code == 200, own.text + assert seen == ["OWN-PARALLEL"] + assert "x-treg-cost-micro" not in own.headers + assert await _balance(clients) == before + finally: + get_settings.cache_clear() diff --git a/tests/test_rankings.py b/tests/test_rankings.py index 99894a622..22ccddd7c 100644 --- a/tests/test_rankings.py +++ b/tests/test_rankings.py @@ -5,6 +5,7 @@ import re from datetime import date, timedelta +import pytest from httpx import AsyncClient from treg.application import rankings as rankings_app @@ -333,8 +334,8 @@ def test_a_model_that_moved_vendor_compares_with_all_its_earlier_calls(): cat.endpoints.remove(ep); cat.by_id.pop(ep["id"]) -def test_a_decision_model_is_its_own_type_and_counts_in_all(): - (jev,) = [e for e in store.load().endpoints if e["platform"] == "ai-judge" and store.browsable(e)] - board = rankings.build(_rows(jev["id"], {0: 50}), store.load(), LAST)["ai_models"]["7d"] +@pytest.mark.parametrize("endpoint_id", ["openrouter.ai-judge.decide", "parallel.ai-judge.decide"]) +def test_a_decision_model_is_its_own_type_and_counts_in_all(endpoint_id): + board = rankings.build(_rows(endpoint_id, {0: 50}), store.load(), LAST)["ai_models"]["7d"] (model,) = board["models"] assert model["type"] == "Decision" and model["calls"] == 50