Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions skills/adversarial-spec/scripts/debate.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
Codex CLI: (ChatGPT subscription) models: codex/gpt-5.2-codex, codex/gpt-5.1-codex-max
Install: npm install -g @openai/codex && codex login
Reasoning: --codex-reasoning xhigh (minimal, low, medium, high, xhigh)
Claude CLI: (Claude account) models: claude-cli/sonnet, claude-cli/opus
Install: npm install -g @anthropic-ai/claude-code && claude

Document types:
prd - Product Requirements Document (business/product focus)
Expand Down
115 changes: 115 additions & 0 deletions skills/adversarial-spec/scripts/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
get_system_prompt,
)
from providers import (
CLAUDE_CLI_AVAILABLE,
CODEX_AVAILABLE,
DEFAULT_CODEX_REASONING,
DEFAULT_COST,
Expand Down Expand Up @@ -370,6 +371,70 @@ def call_codex_model(
raise RuntimeError("Codex CLI not found in PATH")


def call_claude_cli_model(
system_prompt: str,
user_message: str,
model: str,
timeout: int = 600,
) -> tuple[str, int, int]:
"""
Call Claude CLI in print mode.

Args:
system_prompt: System instructions for the model
user_message: User prompt to send
model: Model name (e.g., "claude-cli/sonnet" -> uses "sonnet")
timeout: Timeout in seconds (default 10 minutes)

Returns:
Tuple of (response_text, input_tokens, output_tokens)

Raises:
RuntimeError: If Claude CLI is not available or fails
"""
if not CLAUDE_CLI_AVAILABLE:
raise RuntimeError(
"Claude CLI not found. Install with: npm install -g @anthropic-ai/claude-code"
)

actual_model = model.split("/", 1)[1] if "/" in model else model

try:
cmd = [
"claude",
"-p",
"--output-format",
"text",
"--model",
actual_model,
"--append-system-prompt",
system_prompt,
user_message,
]

result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)

if result.returncode != 0:
error_msg = (
result.stderr.strip() or f"Claude CLI exited with code {result.returncode}"
)
raise RuntimeError(f"Claude CLI failed: {error_msg}")

response_text = result.stdout.strip()
if not response_text:
raise RuntimeError("No response from Claude CLI")

input_tokens = (len(system_prompt) + len(user_message)) // 4
output_tokens = len(response_text) // 4

return response_text, input_tokens, output_tokens

except subprocess.TimeoutExpired:
raise RuntimeError(f"Claude CLI timed out after {timeout}s")
except FileNotFoundError:
raise RuntimeError("Claude CLI not found in PATH")


def call_gemini_cli_model(
system_prompt: str,
user_message: str,
Expand Down Expand Up @@ -604,6 +669,56 @@ def call_single_model(
model=model, response="", agreed=False, spec=None, error=last_error
)

# Route Claude CLI models to dedicated handler
if model.startswith("claude-cli/"):
last_error = None
for attempt in range(MAX_RETRIES):
try:
content, input_tokens, output_tokens = call_claude_cli_model(
system_prompt=system_prompt,
user_message=user_message,
model=model,
timeout=timeout,
)
agreed = "[AGREE]" in content
extracted = extract_spec(content)

if not agreed and not extracted:
print(
f"Warning: {model} provided critique but no [SPEC] tags found. Response may be malformed.",
file=sys.stderr,
)

cost = cost_tracker.add(model, input_tokens, output_tokens)

return ModelResponse(
model=model,
response=content,
agreed=agreed,
spec=extracted,
input_tokens=input_tokens,
output_tokens=output_tokens,
cost=cost,
)
except Exception as e:
last_error = str(e)
if attempt < MAX_RETRIES - 1:
delay = RETRY_BASE_DELAY * (2**attempt)
print(
f"Warning: {model} failed (attempt {attempt + 1}/{MAX_RETRIES}): {last_error}. Retrying in {delay:.1f}s...",
file=sys.stderr,
)
time.sleep(delay)
else:
print(
f"Error: {model} failed after {MAX_RETRIES} attempts: {last_error}",
file=sys.stderr,
)

return ModelResponse(
model=model, response="", agreed=False, spec=None, error=last_error
)

# Standard litellm path for all other providers
last_error = None
display_model = model
Expand Down
41 changes: 38 additions & 3 deletions skills/adversarial-spec/scripts/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@
"codex/gpt-5.2-codex": {"input": 0.0, "output": 0.0},
"codex/gpt-5.1-codex-max": {"input": 0.0, "output": 0.0},
"codex/gpt-5.1-codex-mini": {"input": 0.0, "output": 0.0},
# Claude CLI models (uses Claude account, no per-token cost)
"claude-cli/sonnet": {"input": 0.0, "output": 0.0},
"claude-cli/opus": {"input": 0.0, "output": 0.0},
"claude-cli/haiku": {"input": 0.0, "output": 0.0},
# Gemini CLI models (uses Google account, no per-token cost)
"gemini-cli/gemini-3-pro-preview": {"input": 0.0, "output": 0.0},
"gemini-cli/gemini-3-flash-preview": {"input": 0.0, "output": 0.0},
Expand All @@ -47,6 +51,9 @@
# Check if Codex CLI is available
CODEX_AVAILABLE = shutil.which("codex") is not None

# Check if Claude CLI is available
CLAUDE_CLI_AVAILABLE = shutil.which("claude") is not None

# Check if Gemini CLI is available
GEMINI_CLI_AVAILABLE = shutil.which("gemini") is not None

Expand Down Expand Up @@ -315,6 +322,13 @@ def list_providers():
print(" Install: npm install -g @openai/codex && codex login")
print()

# Claude CLI (uses Claude account, not API key)
claude_cli_status = "[installed]" if CLAUDE_CLI_AVAILABLE else "[not installed]"
print(f" {'Claude CLI':12} {'(Claude account)':24} {claude_cli_status}")
print(" Example models: claude-cli/sonnet, claude-cli/opus")
print(" Install: npm install -g @anthropic-ai/claude-code && claude")
print()

# Gemini CLI (uses Google account, not API key)
gemini_cli_status = "[installed]" if GEMINI_CLI_AVAILABLE else "[not installed]"
print(f" {'Gemini CLI':12} {'(Google account)':24} {gemini_cli_status}")
Expand Down Expand Up @@ -384,6 +398,10 @@ def get_available_providers() -> list[tuple[str, Optional[str], str]]:
if CODEX_AVAILABLE:
available.append(("Codex CLI", None, "codex/gpt-5.2-codex"))

# Add Claude CLI if available
if CLAUDE_CLI_AVAILABLE:
available.append(("Claude CLI", None, "claude-cli/sonnet"))

# Add Gemini CLI if available
if GEMINI_CLI_AVAILABLE:
available.append(("Gemini CLI", None, "gemini-cli/gemini-3-pro-preview"))
Expand All @@ -393,12 +411,12 @@ def get_available_providers() -> list[tuple[str, Optional[str], str]]:

def get_default_model() -> Optional[str]:
"""
Get a default model based on available API keys.
Get a default model list based on available providers.

Checks Bedrock first, then API keys in priority order.
Checks Bedrock first, then prefers local CLI providers, then API keys.

Returns:
Model name string, or None if no API keys are configured.
Comma-separated model list string, or None if no providers are configured.
"""
# Check Bedrock first
bedrock_config = get_bedrock_config()
Expand All @@ -407,6 +425,14 @@ def get_default_model() -> Optional[str]:
if available_models:
return available_models[0]

default_models: list[str] = []
if CODEX_AVAILABLE:
default_models.append("codex/gpt-5.2-codex")
if CLAUDE_CLI_AVAILABLE:
default_models.append("claude-cli/sonnet")
if default_models:
return ",".join(default_models)

# Check API keys
available = get_available_providers()
if available:
Expand Down Expand Up @@ -445,6 +471,7 @@ def validate_model_credentials(models: list[str]) -> tuple[list[str], list[str]]
"deepseek/": "DEEPSEEK_API_KEY",
"zhipu/": "ZHIPUAI_API_KEY",
"codex/": None, # Uses ChatGPT subscription, not API key
"claude-cli/": None, # Uses Claude account, not API key
"gemini-cli/": None, # Uses Google account, not API key
}

Expand All @@ -457,6 +484,14 @@ def validate_model_credentials(models: list[str]) -> tuple[list[str], list[str]]
invalid.append(model)
continue

# Check if it's a Claude CLI model
if model.startswith("claude-cli/"):
if CLAUDE_CLI_AVAILABLE:
valid.append(model)
else:
invalid.append(model)
continue

# Check if it's a Gemini CLI model
if model.startswith("gemini-cli/"):
if GEMINI_CLI_AVAILABLE:
Expand Down
129 changes: 129 additions & 0 deletions skills/adversarial-spec/scripts/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
RETRY_BASE_DELAY,
CostTracker,
ModelResponse,
call_claude_cli_model,
call_codex_model,
call_gemini_cli_model,
call_models_parallel,
Expand Down Expand Up @@ -789,6 +790,92 @@ def test_uses_yolo_flag(self, mock_run):
assert "-y" in cmd


class TestCallClaudeCliModel:
@patch("models.CLAUDE_CLI_AVAILABLE", False)
def test_raises_when_claude_cli_unavailable(self):
import pytest

with pytest.raises(RuntimeError, match="Claude CLI not found"):
call_claude_cli_model("system", "user", "claude-cli/sonnet")

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_extracts_model_name_from_prefix(self, mock_run):
mock_run.return_value = Mock(
returncode=0,
stdout="Test response from Claude",
stderr="",
)
response, inp, out = call_claude_cli_model(
"sys", "user", "claude-cli/sonnet"
)
cmd = mock_run.call_args[0][0]
assert "sonnet" in cmd
assert "claude-cli/sonnet" not in " ".join(cmd)

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_returns_response_text(self, mock_run):
mock_run.return_value = Mock(
returncode=0,
stdout="Test response from Claude",
stderr="",
)
response, inp, out = call_claude_cli_model("sys", "user", "claude-cli/sonnet")
assert response == "Test response from Claude"

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_handles_nonzero_exit_code(self, mock_run):
import pytest

mock_run.return_value = Mock(returncode=1, stdout="", stderr="Some error")
with pytest.raises(RuntimeError, match="Claude CLI failed"):
call_claude_cli_model("sys", "user", "claude-cli/sonnet")

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_raises_on_empty_response(self, mock_run):
import pytest

mock_run.return_value = Mock(returncode=0, stdout="", stderr="")
with pytest.raises(RuntimeError, match="No response from Claude CLI"):
call_claude_cli_model("sys", "user", "claude-cli/sonnet")

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_timeout_raises_runtime_error(self, mock_run):
import subprocess
import pytest

mock_run.side_effect = subprocess.TimeoutExpired("claude", 600)
with pytest.raises(RuntimeError, match="timed out"):
call_claude_cli_model("sys", "user", "claude-cli/sonnet")

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_file_not_found_raises_runtime_error(self, mock_run):
import pytest

mock_run.side_effect = FileNotFoundError()
with pytest.raises(RuntimeError, match="not found in PATH"):
call_claude_cli_model("sys", "user", "claude-cli/sonnet")

@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.subprocess.run")
def test_estimates_tokens(self, mock_run):
mock_run.return_value = Mock(
returncode=0,
stdout="Response text here",
stderr="",
)
response, inp, out = call_claude_cli_model(
"system prompt", "user message", "claude-cli/sonnet"
)
assert inp > 0
assert out > 0


class TestCallSingleModel:
@patch("models.completion")
def test_returns_model_response_on_success(self, mock_completion):
Expand Down Expand Up @@ -1057,6 +1144,48 @@ def test_gemini_cli_exponential_backoff(self, mock_sleep, mock_gemini):
assert calls[0][0][0] == 1.0 # First delay
assert calls[1][0][0] == 2.0 # Second delay

@patch("models.call_claude_cli_model")
@patch("models.CLAUDE_CLI_AVAILABLE", True)
def test_routes_claude_cli_model_to_handler(self, mock_claude):
mock_claude.return_value = ("[AGREE]\n[SPEC]spec[/SPEC]", 100, 50)

result = call_single_model("claude-cli/sonnet", "spec", 1, "prd")
mock_claude.assert_called_once()
assert result.model == "claude-cli/sonnet"

@patch("models.call_claude_cli_model")
@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.time.sleep")
def test_claude_cli_retries_on_failure(self, mock_sleep, mock_claude):
mock_claude.side_effect = [Exception("First fail"), ("[AGREE]", 10, 5)]

result = call_single_model("claude-cli/sonnet", "spec", 1, "prd")
assert mock_claude.call_count == 2
assert result.agreed is True

@patch("models.call_claude_cli_model")
@patch("models.CLAUDE_CLI_AVAILABLE", True)
def test_claude_cli_extracts_spec_from_response(self, mock_claude):
mock_claude.return_value = ("Critique\n[SPEC]Extracted spec[/SPEC]", 100, 50)

result = call_single_model("claude-cli/sonnet", "spec", 1, "prd")
assert result.spec == "Extracted spec"

@patch("models.call_claude_cli_model")
@patch("models.CLAUDE_CLI_AVAILABLE", True)
@patch("models.time.sleep")
def test_claude_cli_exponential_backoff(self, mock_sleep, mock_claude):
mock_claude.side_effect = [
Exception("First fail"),
Exception("Second fail"),
("[AGREE]", 10, 5),
]

call_single_model("claude-cli/sonnet", "spec", 1, "prd")
calls = mock_sleep.call_args_list
assert calls[0][0][0] == 1.0 # First delay
assert calls[1][0][0] == 2.0 # Second delay


class TestCallModelsParallel:
@patch("models.call_single_model")
Expand Down
Loading