Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
78f6339
feat(catalog): add Parallel Decisions tool
wberri13 Oct 30, 2026
efdabea
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 1, 2026
75b60c7
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 1, 2026
7a6bff0
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 2, 2026
08d3053
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 2, 2026
639c8ef
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 2, 2026
71a1268
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 3, 2026
e02e5c6
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 3, 2026
04a035a
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 3, 2026
8664b36
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 4, 2026
de3cba4
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 4, 2026
7ee6aff
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 4, 2026
a0ca50e
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 5, 2026
3ec7bf1
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 5, 2026
02f9a16
chore: merge upstream main into Parallel Decisions PR
wberri13 Nov 5, 2026
832bee6
chore: merge upstream main into Parallel decisions PR
wberri13 Nov 5, 2026
35e2606
chore: merge upstream main into Parallel decisions PR
wberri13 Nov 6, 2026
8971ede
chore: merge upstream main into Parallel decisions PR
wberri13 Nov 7, 2026
29ca3f8
chore: merge upstream main into Parallel decisions PR
wberri13 Nov 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions examples/parallel-decisions.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# Parallel Decisions through treg

Connect your team's Parallel API key once. Calls with this key are billed by Parallel to your
Parallel account and do not debit your treg balance.

```bash
treg secret add parallel --value "$PARALLEL_API_KEY"
treg catalog get parallel.ai-judge.decide
```

One predicate question:

```bash
treg call parallel.ai-judge.decide --data '{
"state": {"company": "Parallel", "website": "https://parallel.ai"},
"questions": {
"has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"}
},
"include_excerpts": false
}'
```

Several questions can share the same state and web evidence in one request:

```bash
treg call parallel.ai-judge.decide --data '{
"state": {"company": "Parallel", "website": "https://parallel.ai"},
"questions": {
"has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"},
"product_type": {
"type": "choice",
"instructions": "Which description best fits Parallel?",
"criteria": {"web_api": "Offers a web API", "desktop_app": "Offers a desktop app"}
},
"documentation_level": {
"type": "score",
"instructions": "How detailed are Parallel's public API docs?",
"criteria": ["No docs", "Partial docs", "Detailed docs"]
}
}
}'
```

Read each result under `answers.<question_name>`. The response includes the vendor's `decision_id`,
`mode`, token `usage`, and shared `citations`. A question can return `type: "refusal"` instead of a
decision value. See [Parallel's question types](https://docs.parallel.ai/decisions/question-types)
for criteria and limits.

## Verification ledger

Checked against Parallel's [API reference](https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions),
[question types](https://docs.parallel.ai/decisions/question-types),
[best practices](https://docs.parallel.ai/decisions/best-practices),
[pricing](https://docs.parallel.ai/getting-started/pricing), and
[rate limits](https://docs.parallel.ai/getting-started/rate-limits) on 2026-10-30.
The documented rate is $0.001 per `fast` request with up to 100 questions.

`tests/test_parallel_decisions.py` uses a mocked upstream for single and multiple questions,
response fields and citations, upstream error statuses, own-key precedence, and the platform-key
gate. `src/treg/catalog/examples/parallel.ai-judge.decide.json` is synthetic and sanitized; it is
not a captured live response. No live Parallel Decisions call or vendor billing observation was
made. Platform-funded access remains disabled until a maintainer verifies the platform key can
call Decisions, confirms actual charges and error billing against the shared account, and selects
settlement behavior from that evidence.
25 changes: 25 additions & 0 deletions src/treg/catalog/examples/parallel.ai-judge.decide.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
{
"mode": "fast",
"answers": {
"has_api": {"type": "noul", "noul": 0.99},
"product_type": {
"type": "choice",
"choice": "web_api",
"probabilities": {"web_api": 0.95, "desktop_app": 0.05},
"confidence": 0.91
},
"documentation_level": {
"type": "score",
"score": 1.8,
"legend": {"0": "No public documentation", "1": "Partial documentation", "2": "Detailed documentation"},
"probabilities": {"0": 0.0, "1": 0.2, "2": 0.8},
"confidence": 0.86
},
"unanswerable": {"type": "refusal"}
},
"usage": {"input_tokens": 1200, "output_tokens": 0},
"citations": [
{"url": "https://example.com/docs", "title": "Example documentation", "excerpts": ["Example excerpt."]}
],
"decision_id": "decisions_00000000000000000000000000000000"
}
37 changes: 35 additions & 2 deletions src/treg/catalog/parallel.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,44 @@ provider: parallel
source:
docs: https://docs.parallel.ai/getting-started/overview
openapi: https://docs.parallel.ai/public-openapi.json
curated: '2026-10-09'
curated: '2026-10-30'
pricing_url: https://docs.parallel.ai/getting-started/pricing
limits: "Search, Extract and Entity Search each allow 600 requests per minute on a rolling 60-second window. HTTP 429 is a rate limit; HTTP 402 is insufficient credit, which in-flight requests count against."
limits: "Search, Extract, Entity Search and Decisions each allow 600 requests per minute on a rolling 60-second window. HTTP 429 is a rate limit; HTTP 402 is insufficient credit, which in-flight requests count against."

endpoints:
- id: parallel.ai-judge.decide
capability: ai-judge.decide
platform: ai-judge
domain: decisions
scope: any_account
method: POST
path: /v1/decisions
platform_blocked: "Platform-key billing and settlement for Decisions have not been live-verified; connect your team's own Parallel key."
name: Make web-grounded decisions
summary: Answer up to 100 named predicate, choice or score questions using shared state and current web evidence.
input:
bodyType: json
body:
state: {type: any, required: true, example: {company: "Parallel", website: "https://parallel.ai"}, note: "Any JSON value shared by all questions. State plus the largest question is limited to 32,000 estimated input tokens."}
questions: {type: object, required: true, example: {has_api: {type: noul, instructions: "Does Parallel offer an API?"}}, note: "1–100 names matching [A-Za-z0-9_-]+. Each value has type noul, choice or score and instructions. Noul may include criteria {true, false}; choice requires 2–255 named option descriptions; score requires 2–10 ordered level descriptions. Each rendered question is limited to 5,000 characters."}
mode: {type: string, required: false, default: fast, enum: [fast], note: "Only fast mode is currently supported."}
include_excerpts: {type: boolean, required: false, default: true, note: "False keeps citation URLs and titles but returns empty excerpts arrays."}
note: "Answers are keyed by question name. Noul returns a probability; choice returns the chosen key and option probabilities; score returns a weighted mean, legend and probabilities. Choice and score may include confidence. Any answer may be a refusal. The raw response also includes mode, decision_id, token usage and citations shared across all questions; citations are not mapped to individual answers."
test_request: {body: {state: {company: "Parallel", website: "https://parallel.ai"}, questions: {has_api: {type: noul, instructions: "Does Parallel offer an API?"}}, include_excerpts: false}}
cost:
type: per_call
value: 0.001
currency: USD
unit: call
source: docs
source_url: https://docs.parallel.ai/getting-started/pricing
checked: '2026-10-30'
confidence: documented
note: "$0.001 per fast request, including up to 100 questions. Documented rate only; platform-key billing and settlement remain unverified, so this tool is team-key only. The response's usage is token counts, not a charge."
added: '2026-10-30'
example_response: examples/parallel.ai-judge.decide.json
docs_url: https://docs.parallel.ai/decisions/decisions-quickstart

- id: parallel.web.search
capability: web.search
platform: web
Expand Down
165 changes: 165 additions & 0 deletions tests/test_parallel_decisions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
"""Parallel Decisions catalog contract and faithful own-key relay (mocked upstream)."""

import json
from pathlib import Path

import httpx
import pytest

from treg import api as A
from treg.config import get_settings
from treg.domain.catalog import store as catalog_store


ENDPOINT = "parallel.ai-judge.decide"
FIXTURE = (Path(__file__).resolve().parents[1] / "src/treg/catalog/examples"
/ f"{ENDPOINT}.json")


class _JSONStream(httpx.AsyncByteStream):
def __init__(self, payload):
self.payload = json.dumps(payload).encode()

async def __aiter__(self):
yield self.payload


def _response(status, payload, *, headers=None):
return httpx.Response(status, headers={"content-type": "application/json", **(headers or {})},
stream=_JSONStream(payload))


async def _balance(clients):
org_id = (await clients.get("/orgs")).json()[0]["org_id"]
return (await clients.get(f"/orgs/{org_id}/balance")).json()["balance_micro"]


def test_decisions_catalog_contract():
catalog = catalog_store.load()
endpoint = catalog.by_id[ENDPOINT]
assert endpoint["capability"] == "ai-judge.decide"
assert (endpoint["method"], endpoint["path"]) == ("POST", "/v1/decisions")
assert endpoint["input"]["body"]["state"]["type"] == "any"
assert endpoint["input"]["body"]["questions"]["type"] == "object"
assert endpoint["input"]["body"]["include_excerpts"]["default"] is True
assert endpoint["cost"]["value"] == 0.001
assert endpoint["cost"]["confidence"] == "documented"
assert not catalog.platform_eligible(endpoint)
response = json.loads(FIXTURE.read_text())
assert {"noul", "choice", "score", "refusal"} == {
answer["type"] for answer in response["answers"].values()
}


async def test_decisions_is_discoverable(clients):
response = await clients.get(f"/catalog/endpoints/{ENDPOINT}")
assert response.status_code == 200
assert response.json()["endpoint"]["id"] == ENDPOINT
assert response.json()["endpoint"]["platform_eligible"] is False
assert response.json()["example_response"]["decision_id"].startswith("decisions_")


@pytest.mark.parametrize("state", [None, "Parallel", {"company": "Parallel"}])
async def test_decisions_single_question_preserves_json_state_and_response(clients, monkeypatch, state):
secret = await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"})
assert secret.status_code == 200, secret.text
request_body = {
"state": state,
"questions": {"has_api": {"type": "noul", "instructions": "Does Parallel offer an API?"}},
"include_excerpts": False,
}
reply = json.loads(FIXTURE.read_text())
seen = []

def serve(request):
seen.append((request.url.path, request.headers.get("x-api-key"), json.loads(request.content)))
return _response(200, reply)

before = await _balance(clients)
async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream:
monkeypatch.setattr(A.app.state, "http", upstream)
response = await clients.post(f"/call/{ENDPOINT}", json=request_body)
assert response.status_code == 200, response.text
assert response.json() == reply
assert seen == [("/v1/decisions", "OWN-PARALLEL", request_body)]
assert "x-treg-cost-micro" not in response.headers
assert await _balance(clients) == before


async def test_decisions_multiple_question_types_and_citations_are_relayed(clients, monkeypatch):
await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"})
body = {
"state": {"company": "Parallel", "website": "https://parallel.ai"},
"mode": "fast",
"questions": {
"has_api": {"type": "noul", "instructions": "Does Parallel offer an API?",
"criteria": {"true": "There is an API.", "false": "There is no API."}},
"product_type": {"type": "choice", "instructions": "What kind of product is it?",
"criteria": {"web_api": "A web API", "desktop_app": "A desktop app"}},
"documentation_level": {"type": "score", "instructions": "How detailed are the docs?",
"criteria": ["None", "Partial", "Detailed"]},
},
}
reply = json.loads(FIXTURE.read_text())

def serve(request):
assert request.headers["x-api-key"] == "OWN-PARALLEL"
assert json.loads(request.content) == body
return _response(200, reply)

async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream:
monkeypatch.setattr(A.app.state, "http", upstream)
response = await clients.post(f"/call/{ENDPOINT}", json=body)
assert response.status_code == 200, response.text
assert response.json() == reply
assert response.json()["answers"]["documentation_level"]["confidence"] == 0.86
assert response.json()["citations"][0]["excerpts"] == ["Example excerpt."]


@pytest.mark.parametrize("status", [401, 422, 429, 503])
async def test_decisions_upstream_errors_follow_the_relay(clients, monkeypatch, status):
await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"})
error = {"type": "error", "error": {"ref_id": "example", "message": "Example error"}}

def serve(request):
return _response(status, error, headers={"retry-after": "3"})

before = await _balance(clients)
async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream:
monkeypatch.setattr(A.app.state, "http", upstream)
response = await clients.post(f"/call/{ENDPOINT}", json={
"state": {}, "questions": {"q": {"type": "noul", "instructions": "Is it true?"}},
})
assert response.status_code == status, response.text
assert response.json() == error
assert await _balance(clients) == before


async def test_decisions_never_uses_platform_key_until_billing_is_verified(clients, monkeypatch):
monkeypatch.setenv("TREG_PLATFORM_KEY_PARALLEL", "PLATFORM-PARALLEL")
monkeypatch.setenv("TREG_PLATFORM_PROVIDERS", "parallel")
get_settings.cache_clear()
seen = []
body = {"state": {}, "questions": {"q": {"type": "noul", "instructions": "Is it true?"}}}

def serve(request):
seen.append(request.headers.get("x-api-key"))
return _response(200, json.loads(FIXTURE.read_text()))

try:
before = await _balance(clients)
async with httpx.AsyncClient(transport=httpx.MockTransport(serve)) as upstream:
monkeypatch.setattr(A.app.state, "http", upstream)
missing = await clients.post(f"/call/{ENDPOINT}", json=body)
assert missing.status_code == 404
assert seen == []
assert await _balance(clients) == before

await clients.post("/secrets", json={"name": "parallel", "value": "OWN-PARALLEL"})
own = await clients.post(f"/call/{ENDPOINT}", json=body)
assert own.status_code == 200, own.text
assert seen == ["OWN-PARALLEL"]
assert "x-treg-cost-micro" not in own.headers
assert await _balance(clients) == before
finally:
get_settings.cache_clear()
7 changes: 4 additions & 3 deletions tests/test_rankings.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import re
from datetime import date, timedelta

import pytest
from httpx import AsyncClient

from treg.application import rankings as rankings_app
Expand Down Expand Up @@ -333,8 +334,8 @@ def test_a_model_that_moved_vendor_compares_with_all_its_earlier_calls():
cat.endpoints.remove(ep); cat.by_id.pop(ep["id"])


def test_a_decision_model_is_its_own_type_and_counts_in_all():
(jev,) = [e for e in store.load().endpoints if e["platform"] == "ai-judge" and store.browsable(e)]
board = rankings.build(_rows(jev["id"], {0: 50}), store.load(), LAST)["ai_models"]["7d"]
@pytest.mark.parametrize("endpoint_id", ["openrouter.ai-judge.decide", "parallel.ai-judge.decide"])
def test_a_decision_model_is_its_own_type_and_counts_in_all(endpoint_id):
board = rankings.build(_rows(endpoint_id, {0: 50}), store.load(), LAST)["ai_models"]["7d"]
(model,) = board["models"]
assert model["type"] == "Decision" and model["calls"] == 50
Loading