diff --git a/.gitignore b/.gitignore index c22b62dae..9344c978e 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,8 @@ remotion-composer/public/* remotion-composer/public/demo-props/test-* remotion-composer/public/demo-props/talking-head-* remotion-composer/public/demo-props/caption-burn-* + +# Backup files of .env (auto-created) +.env.backup-* +.env.bak +.claude-flow/ diff --git a/WORKSPACE.md b/WORKSPACE.md new file mode 100644 index 000000000..e7469ba48 --- /dev/null +++ b/WORKSPACE.md @@ -0,0 +1,132 @@ +# Workspace — Semih's OpenMontage Action Hub + +> **Bu dosya senin için, OSS değil.** OpenMontage upstream pull yaparsan untracked görünür — push etmediğin sürece sorun yok. +> **Upstream CLAUDE.md → AGENT_GUIDE.md** zorunlu sıra (Rule Zero) hâlâ geçerli; bu dosya o akışın **üstüne** eklenir, yerine geçmez. + +--- + +## Hızlı Karar Ağacı (yeni Claude session açtığında) + +``` +Sen ne yapmak istiyorsun? +│ +├─ " için OpenMontage kurulumu yapacağız" (TRIGGER PHRASE) +│ " için yeni client setup" +│ "Yeni proje aç " +│ "/onboard " +│ → internal/CLIENT_ONBOARDING.md protokolü çalıştır (6-step) +│ → internal/scripts/new-client.sh "" otomatik scaffolder +│ +├─ "Bir client için video üreteceğim" (workspace zaten var) +│ → internal/SERVICE_PLAYBOOK.md (intake → delivery, 7 aşama) +│ +├─ "Sıfırdan kendi videom için brief vereceğim" +│ → Mode 1: claude → "use hybrid pipeline, [brief]" (chat ile) +│ +├─ "Hızlı zero-key demo ile sistem testi" +│ → make demo ($0, ~1 dk) +│ +├─ "Reference video klonlamak istiyorum" +│ → ~/.claude/skills/video-clone-system.md skill'i tetikle +│ → ~/.claude/scripts/video-clone-analyze.py +│ +├─ "Visual preview / Remotion ile düzenle" +│ → Mode 3: cd remotion-composer && npx remotion studio +│ +└─ "Bilgi/komut hatırlamadım, lookup lazım" + → internal/QUICKSTART.md (cheat sheet) + → internal/KNOWLEDGE_INDEX.md (her şeyin map'i) +``` + +## ⚡ Trigger Phrase: Client Onboarding + +Kullanıcı bu cümlelerden birini söyleyince Claude **otomatik** `internal/CLIENT_ONBOARDING.md` protokolünü çalıştırır: + +| Trigger | Aksiyon | +|---|---| +| "XYZ için OpenMontage kurulumu yapacağız" | new-client.sh + intake.md aç | +| " için yeni client setup" | Aynı | +| "Yeni iş geldi, için hazırlık" | Aynı | +| "Client onboard et: " | Aynı | +| "/onboard " | Aynı | + +Detaylı protokol: `internal/CLIENT_ONBOARDING.md` (6 adım: scaffold → intake → preflight → reference → meta → pipeline-suggest). + +--- + +## İlk Aktivasyon (her session) + +```bash +cd ~/projects/OpenMontage && source .venv/bin/activate +make preflight # tool envanteri sağlık check +``` + +`preflight` çıktısında 9/9 composition tool görmen lazım. Görmüyorsan `make setup` koş. + +--- + +## Knowledge Stack (cross-link map) + +``` +Bu dosya (WORKSPACE.md) + └─ internal/ ← TÜM kullanıcı dökümanları burada (gitignored) + ├─ README.md ← klasör haritası + ├─ CHANGELOG.md ← setup geçmişi (her session'ın özeti) + ├─ KNOWLEDGE_INDEX.md ← her dosyaya lookup + ├─ QUICKSTART.md ← komut cheat sheet + ├─ SERVICE_PLAYBOOK.md ← client iş akışı (intake → delivery) + ├─ CLIENT_ONBOARDING.md ← "X için kurulum" trigger protokolü + ├─ ROADMAP.md ← sonraki adımlar + ├─ scripts/new-client.sh ← workspace scaffolder + ├─ templates/client-intake.md ← intake form template + └─ research/ ← araştırma notları + patch'ler + ├─ INDEX.md + ├─ openmontage-prompt-postprocess-fix.md + ├─ openmontage-video-prompt-button.md + ├─ sample-artifacts-{brief,script}.json + └─ archive/ ← eski büyük roadmap'ler + +Upstream OSS (DOKUNMA): + ├─ AGENT_GUIDE.md ← Rule Zero + ├─ pipeline_defs/{hybrid,cinematic,animated-explainer}.yaml + ├─ skills/creative/video-gen-prompting.md ← universal 5-aspect + ├─ skills/creative/prompting/*.md ← per-provider grammar + └─ lib/shot_prompt_builder.py:82-144 ← deterministik builder + +Senin custom OpenMontage uzantıların (gitignored değil ama upstream'le çakışmaz): + ├─ tools/audio/qwen3_tts.py + ├─ tools/graphics/kie_nano_banana.py + ├─ tools/graphics/kie_gpt_image.py + ├─ tools/video/kie_seedance.py + ├─ tools/video/kie_kling.py + └─ lib/kie_client.py + +Memory (her Claude session'da auto-load): + ├─ ~/.claude/projects/-Users-abalioglu/memory/skill_openmontage.md + ├─ ~/.claude/projects/-Users-abalioglu/memory/skill_openmontage_advanced.md + ├─ ~/.claude/projects/-Users-abalioglu/memory/skill_openmontage_client_onboarding.md + └─ ~/.claude/projects/-Users-abalioglu/memory/project_openmontage_service_offering.md +``` + +--- + +## Servis Modeli — Kısaca + +**Pivot (2026-05-07):** AdSwap SaaS pause edildi. Yeni yön: **e-ticaret markalarına video prodüksiyon servisi** (agency model, not SaaS). + +- Hedef pazar: Shopify/WooCommerce dropship + DTC beauty/cosmetics +- Pricing fikri: UGC starter $300, brand cinematic $800, full campaign $1500 +- Edge: OpenMontage 9-katman + Seedance multi-shot identity + character consistency protocol — kullanıcı sadece brief verir, biz prompt + render + compose orchestrasyonunu yaparız +- Detay: `internal/SERVICE_PLAYBOOK.md` + +--- + +## Bugünün Durumu + +- ✅ OpenMontage kurulu, .venv izole, .env API key'leri kontrolü gerekiyor (FAL/OpenAI/ElevenLabs/Google) +- ✅ 3 demo render (sağlık testi geçti — `projects/demos/renders/`) +- ✅ Prompt engineering insights belgelendi (memory + Documents) +- ✅ Surgical post-process patch yazıldı (Desktop + Documents) — diğer Claude session'a paslanmaya hazır +- ⏳ Sample portfolio üretimi (3-5 sektör) +- ⏳ Pricing landing page (agentized.io altına) +- ⏳ İlk client outreach diff --git a/lib/kie_client.py b/lib/kie_client.py new file mode 100644 index 000000000..082b9ccfc --- /dev/null +++ b/lib/kie_client.py @@ -0,0 +1,281 @@ +"""Shared KIE.AI HTTP client. + +Used by tools/graphics/kie_*.py and tools/video/kie_*.py. + +Reference: ~/.claude/projects/-Users-abalioglu/memory/reference_kieai_models.md +- Pattern A (Unified Market API): POST /api/v1/jobs/createTask + GET /api/v1/jobs/recordInfo +- Pattern B (Dedicated APIs): each model family has its own create+query endpoints + +This client implements Pattern A (covers ~80% of the catalog) plus thin helpers +for the dedicated endpoints we use (Veo, 4o Image). + +Auth: `Authorization: Bearer ` +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import requests + + +KIE_BASE = "https://api.kie.ai/api/v1" +KIE_FILE_BASE = "https://kieai.redpandaai.co/api" + + +class KIEError(RuntimeError): + """Raised on KIE API errors.""" + + +def get_api_key() -> str | None: + return os.environ.get("KIE_AI_API_KEY") or os.environ.get("KIEAI_API_KEY") + + +def is_configured() -> bool: + return bool(get_api_key()) + + +def _headers(extra: dict[str, str] | None = None) -> dict[str, str]: + api_key = get_api_key() + if not api_key: + raise KIEError("KIE_AI_API_KEY not set") + h = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + if extra: + h.update(extra) + return h + + +# ── Pattern A: Unified Market API ──────────────────────────────────────── + + +def create_task(model: str, input_payload: dict[str, Any], *, timeout: int = 30) -> str: + """POST /api/v1/jobs/createTask — returns taskId. + + `model` is the KIE model identifier, e.g.: + - "google/nano-banana" + - "nano-banana-2" + - "openai/gpt-image-2" + - "bytedance/seedance-2-fast" + - "kling-2.6/image-to-video" + - "kling-3.0/video" + """ + body = {"model": model, "input": input_payload} + r = requests.post(f"{KIE_BASE}/jobs/createTask", json=body, headers=_headers(), timeout=timeout) + if r.status_code >= 400: + raise KIEError(f"createTask {r.status_code}: {r.text[:500]}") + j = r.json() + if not j.get("data") and not j.get("taskId"): + raise KIEError(f"createTask response missing taskId: {j}") + return j.get("data", {}).get("taskId") or j.get("taskId") + + +def poll_record(task_id: str, *, timeout: int = 30) -> dict[str, Any]: + """GET /api/v1/jobs/recordInfo?taskId=… — returns the record dict. + + States: waiting → queuing → generating → success | fail + """ + r = requests.get( + f"{KIE_BASE}/jobs/recordInfo", + params={"taskId": task_id}, + headers=_headers(), + timeout=timeout, + ) + if r.status_code >= 400: + raise KIEError(f"recordInfo {r.status_code}: {r.text[:500]}") + j = r.json() + return j.get("data") or j + + +def wait_for_completion( + task_id: str, + *, + poll_interval_s: float = 4.0, + max_wait_s: float = 600.0, +) -> dict[str, Any]: + """Block until task is `success` or `fail`. Returns the final record. + + Raises KIEError on `fail` or timeout. + """ + deadline = time.time() + max_wait_s + while time.time() < deadline: + rec = poll_record(task_id) + state = (rec.get("state") or rec.get("status") or "").lower() + if state == "success": + return rec + if state == "fail" or state == "failed": + err = rec.get("failMsg") or rec.get("error") or rec.get("message") or "unknown" + raise KIEError(f"task {task_id} failed: {err}") + time.sleep(poll_interval_s) + raise KIEError(f"task {task_id} timed out after {max_wait_s}s") + + +def run_unified(model: str, input_payload: dict[str, Any], **wait_kwargs) -> dict[str, Any]: + """Convenience: createTask + wait_for_completion in one call. + + Returns the final record dict; result URLs/data are typically inside + `record["resultUrls"]`, `record["output"]`, or `record["data"]` — + structure varies by model family. Caller should know what to extract. + """ + task_id = create_task(model, input_payload) + return wait_for_completion(task_id, **wait_kwargs) + + +# ── Pattern B: Dedicated 4o Image API (alternative GPT image entry) ────── + + +def gpt4o_image_generate(prompt: str, *, n: int = 1, size: str = "1024x1024", **extra) -> str: + """POST /api/v1/gpt4o-image/generate — alternative to Pattern-A `openai/gpt-image-2`. + + Returns taskId. + """ + body = {"prompt": prompt, "n": n, "size": size, **extra} + r = requests.post( + f"{KIE_BASE}/gpt4o-image/generate", + json=body, + headers=_headers(), + timeout=30, + ) + if r.status_code >= 400: + raise KIEError(f"gpt4o-image/generate {r.status_code}: {r.text[:500]}") + j = r.json() + return j.get("data", {}).get("taskId") or j.get("taskId") + + +def gpt4o_image_record(task_id: str) -> dict[str, Any]: + r = requests.get( + f"{KIE_BASE}/gpt4o-image/record-info", + params={"taskId": task_id}, + headers=_headers(), + timeout=30, + ) + if r.status_code >= 400: + raise KIEError(f"gpt4o-image/record-info {r.status_code}: {r.text[:500]}") + return r.json().get("data") or r.json() + + +# ── Helpers: download artifact + upload local file ──────────────────────── + + +def download_to(url: str, dest: Path, *, timeout: int = 120) -> Path: + """Stream-download a result URL to local disk.""" + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + with requests.get(url, stream=True, timeout=timeout) as r: + if r.status_code >= 400: + raise KIEError(f"download {url} → {r.status_code}") + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=64 * 1024): + if chunk: + f.write(chunk) + return dest + + +def upload_file_url(url: str, *, timeout: int = 60) -> str: + """POST kieai.redpandaai.co/api/file-url-upload — returns hosted URL. + + Useful when KIE expects a public URL but you have a local file already + uploaded somewhere (S3/R2/etc). For raw local files use `upload_file_stream`. + """ + r = requests.post( + f"{KIE_FILE_BASE}/file-url-upload", + json={"url": url}, + headers=_headers(), + timeout=timeout, + ) + if r.status_code >= 400: + raise KIEError(f"file-url-upload {r.status_code}: {r.text[:500]}") + j = r.json() + return j.get("data", {}).get("url") or j.get("url") + + +def upload_file_stream(path: str | Path, *, timeout: int = 300) -> str: + """POST kieai.redpandaai.co/api/file-stream-upload — multipart, returns hosted URL. + + Max 100MB. Files are kept for 3 days. + """ + path = Path(path) + if not path.exists(): + raise KIEError(f"local file not found: {path}") + api_key = get_api_key() + if not api_key: + raise KIEError("KIE_AI_API_KEY not set") + with open(path, "rb") as f: + r = requests.post( + f"{KIE_FILE_BASE}/file-stream-upload", + headers={"Authorization": f"Bearer {api_key}"}, + files={"file": (path.name, f)}, + timeout=timeout, + ) + if r.status_code >= 400: + raise KIEError(f"file-stream-upload {r.status_code}: {r.text[:500]}") + j = r.json() + return j.get("data", {}).get("url") or j.get("url") + + +def maybe_upload(path_or_url: str) -> str: + """If `path_or_url` is a URL, return as-is. If it's a local path, upload + return hosted URL.""" + parsed = urlparse(path_or_url) + if parsed.scheme in ("http", "https"): + return path_or_url + return upload_file_stream(path_or_url) + + +# ── Result extraction (varies by model family) ─────────────────────────── + + +def extract_result_urls(record: dict[str, Any]) -> list[str]: + """Extract result URLs from a completed KIE record. + + KIE puts artifacts under different keys depending on the model family: + - Most unified-API models (nano-banana, seedance, kling, etc.) return them + inside `resultJson` as a JSON-encoded STRING containing `{"resultUrls": [...]}`. + - Some return `resultUrls` directly at the top level. + - Some put them under `output` / `data` / `urls`. + + This function tries all known shapes. + """ + import json as _json + + # 1. resultJson — JSON-encoded string (most common for unified API) + rj = record.get("resultJson") + if isinstance(rj, str) and rj.strip(): + try: + parsed = _json.loads(rj) + if isinstance(parsed, dict): + for key in ("resultUrls", "result_urls", "urls", "image_urls", "video_urls", "output_urls"): + v = parsed.get(key) + if v: + return [str(u) for u in (v if isinstance(v, list) else [v]) if u] + except _json.JSONDecodeError: + pass + elif isinstance(rj, dict): + # already-parsed shape + for key in ("resultUrls", "result_urls", "urls", "image_urls", "video_urls", "output_urls"): + v = rj.get(key) + if v: + return [str(u) for u in (v if isinstance(v, list) else [v]) if u] + + # 2. Top-level fields + for key in ("resultUrls", "result_urls", "outputUrls", "output_urls", "urls"): + v = record.get(key) + if v: + return [str(u) for u in (v if isinstance(v, list) else [v]) if u] + + # 3. Nested 'output' or 'data' + nested = record.get("output") or record.get("data") or {} + if isinstance(nested, dict): + for key in ("urls", "result_urls", "resultUrls", "image_urls", "video_urls"): + v = nested.get(key) + if v: + return [str(u) for u in (v if isinstance(v, list) else [v]) if u] + elif isinstance(nested, list): + return [str(u) for u in nested if u] + + return [] diff --git a/skills/creative/image-provider-usage.md b/skills/creative/image-provider-usage.md index 179e64c0e..950c2056b 100644 --- a/skills/creative/image-provider-usage.md +++ b/skills/creative/image-provider-usage.md @@ -10,6 +10,8 @@ | Tool | Provider | Cost | Speed | Best For | |------|----------|------|-------|----------| | `flux_image` | FLUX 2 Pro via fal.ai | ~$0.03-0.05 | ~5-10s | Photorealism, general purpose, workhorse | +| `gemini_image` | Gemini 2.5 Flash Image / 3 Pro Image Preview (Google) | ~$0.04 (Flash) / ~$0.10 (3 Pro) | ~5-15s | Iterative variants (Flash); precision text-heavy / brand-fidelity / strict prompt adherence (3 Pro) | +| `google_imagen` | Imagen 4.0 (Google) | varies by tier | ~5-10s | Photorealism via Imagen family (separate from Gemini native image) | | `grok_image` | Grok Imagine Image (xAI) | $0.02/output + $0.002/input edit image | ~5-15s | Image edits, style transfer, multi-image compositing | | `openai_image` | GPT Image 1 (OpenAI) | ~$0.01-0.17 | ~5-15s | Complex instructions, text in images, multi-element | | `recraft_image` | Recraft V4 via fal.ai | ~$0.04-0.25 | ~5-10s | Logos, SVG vectors, brand assets, text rendering (see caveat below) | @@ -39,8 +41,9 @@ | **Style transfer / repaint of an existing image** | `grok_image` | Native edit flow, strong promptable transforms | `openai_image` | | **Multi-image merge / composite** | `grok_image` | Can combine multiple source images into one scene | `openai_image` | | **Logo or brand asset** | `recraft_image` | SVG support, text accuracy | `openai_image` | -| **Image with text/labels** | `openai_image` | Best text rendering (GPT Image 1) | `recraft_image` | -| **Complex multi-element composition** | `openai_image` | Best instruction following | `flux_image` | +| **Image with text/labels** | `gemini_image` (model=`gemini-3-pro-image-preview`) | Newest precision text rendering | `openai_image` (GPT Image 1) → `recraft_image` | +| **Complex multi-element composition** | `gemini_image` (model=`gemini-3-pro-image-preview`) | Strict prompt adherence on multi-constraint prompts | `openai_image` → `flux_image` | +| **Fast iterative variants of a brand asset** | `gemini_image` (model=`gemini-2.5-flash-image`, default) | Cheap + fast for "give me 4 takes on this concept" | `flux_image` | | **Hero image (key visual)** | `flux_image` | Highest visual quality | `openai_image` | | **Thumbnail** | `flux_image` or `recraft_image` | Needs to be eye-catching | — | | **Budget/free project** | `pexels_image` or `pixabay_image` | Free, immediate | `local_diffusion` | @@ -52,6 +55,12 @@ - **`style` parameter causes 422 errors** (as of 2026-04). The `style` enum values (`digital_illustration`, `realistic_image`, etc.) are rejected by fal.ai's Recraft V4 endpoint. **Workaround:** encode style direction in the prompt text instead (e.g. "digital illustration of a tooth cross-section" rather than `style="digital_illustration"`). The `image_size` and `colors` parameters work fine. - **Text rendering is unreliable for exact business names.** Recraft (like all AI image models) may hallucinate wrong text. For any scene where text must be verbatim (CTA screens, business names, phone numbers), use Remotion `text_card` instead of generating an image with text. +### Gemini Image (`gemini_image`, added 2026-05-08) +- **Two-tier model:** default `gemini-2.5-flash-image` (~$0.04/image, fast iteration); upgrade to `gemini-3-pro-image-preview` (~$0.10/image) for text-heavy / strict-composition / brand-fidelity work. Mirrors the OpenSwarm pattern of using Flash for variants and 3 Pro for precision. +- **Aspect ratios:** broader than OpenAI (`1:1`, `2:3`, `3:2`, `3:4`, `4:3`, `4:5`, `5:4`, `9:16`, `16:9`, `21:9`). +- **Pair with QC auto-fix loop** — see `image-qc-autofix.md`. The recommended pattern is: generate with `gemini-2.5-flash-image` → run QC → if any check fails, auto-retry once with `gemini-3-pro-image-preview` for the precision pass. +- **Distinct from `google_imagen`** — that tool covers the Imagen 4.0 family (`imagen-4.0-generate-001`, `*-ultra`, `*-fast`); `gemini_image` covers Gemini's native multimodal image generation. + ## Cost-Quality Tradeoff ``` diff --git a/skills/creative/image-qc-autofix.md b/skills/creative/image-qc-autofix.md new file mode 100644 index 000000000..d6325f4f6 --- /dev/null +++ b/skills/creative/image-qc-autofix.md @@ -0,0 +1,138 @@ +--- +name: image-qc-autofix +layer: 2 +status: optional-postprocess +added: 2026-05-08 +ported_from: VRSEN/OpenSwarm image_generation_agent (commit 28e5fe38) +related: image-provider-usage.md, image-gen-usage.md +--- + +# Image QC Auto-Fix Loop + +> **What this is:** A standardized post-generation quality-control pattern. Run after any image-generation tool call — `gemini_image`, `flux_image`, `openai_image`, `recraft_image`, `grok_image`, `local_diffusion`, `google_imagen`. Catches common defects (composition drift, missing elements, broken text, scale issues, lighting artifacts, hallucinated content) and triggers exactly **one** corrective regeneration before final delivery. + +> **Why it exists:** OpenMontage already has Layer-1 EP gating + checkpoint protocol for full pipelines. But for **single-shot image generation calls** outside a pipeline (one-off product shots, blog hero, slide image, asset for an existing video), there's no automated quality gate. This skill fills that gap with a 60-second QC ritual. + +> **When to use:** Any image generation that will be **delivered to a client** or **used as a reference for downstream video generation**. Skip for throw-away tests or agent-internal scratch images. + +--- + +## The Loop + +``` +┌─────────────────────────────────────────────────────────┐ +│ 1. GENERATE gemini_image / flux_image / etc. │ +│ 2. QC PASS 5-bullet checklist (see below) │ +│ 3. PASS? Yes → deliver. No → fix once. │ +│ 4. FIX Same prompt, smarter model │ +│ 5. QC PASS #2 Repeat checklist │ +│ 6. STILL FAILING? Stop. Report failures + 1 next move. │ +└─────────────────────────────────────────────────────────┘ +``` + +**Hard rule:** Maximum **two** generations per QC loop (initial + 1 fix). Do not loop forever — that's how budgets explode. + +--- + +## Step 1: Generate + +Pick a provider per `image-provider-usage.md`. For new work, default to `gemini_image` with `gemini-2.5-flash-image` (cheap + fast for the first take). + +## Step 2: QC Pass — 5-Bullet Checklist + +Look at the rendered image as if a client just sent it back asking "What's wrong with this?" + +Tick each bullet **PASS / FAIL** with one-line evidence: + +1. **Composition match** — does the framing, subject placement, and aspect ratio match the prompt? (PASS / FAIL: "subject is centered but prompt asked for rule-of-thirds left") +2. **Required elements present** — every named entity in the prompt visible? (PASS / FAIL: "missing the second figure described in the prompt") +3. **Text fidelity** — if any text/labels were specified, are they spelled correctly and readable? (PASS / FAIL: "the word 'PREMIUM' came out as 'PREMUM'") +4. **Lighting + color** — does the lighting/palette/mood match the prompt? Any hot spots, banding, or color shifts? (PASS / FAIL: "shadows are too harsh, prompt asked for soft diffused light") +5. **Artifacts + anatomy** — extra fingers, distorted faces, illegible patterns, weird edges? (PASS / FAIL: "hand has 6 fingers") + +**Decision rule:** All 5 PASS → deliver. **Any FAIL** → go to Step 3. + +## Step 3: One Auto-Fix Pass + +The fix should be **smarter, not just retried**. Strategy: + +| Failure type | Fix strategy | +|--------------|--------------| +| Composition / scale | Same provider, augment prompt with explicit framing ("centered, rule-of-thirds left, 3/4 view"). Re-run. | +| Missing element | Same provider, prepend element with "MUST INCLUDE: ..." phrasing. Re-run. | +| Text fidelity broken | **Upgrade to `gemini-3-pro-image-preview` or `openai_image` (GPT Image 1)** — both have superior text rendering. | +| Multi-constraint adherence | **Upgrade to `gemini-3-pro-image-preview`** — best instruction following. | +| Anatomy / artifact | Same provider with negative prompt ("avoid extra fingers, distorted hands") OR upgrade to `flux_image` for photorealism. | +| Style mismatch | Same provider, encode style explicitly in prompt text (don't rely on enum). | +| All-around poor | Upgrade tier (Flash → 3 Pro) OR switch family (Gemini → OpenAI / Flux). | + +**Cost note:** A single upgrade from `gemini-2.5-flash-image` (~$0.04) to `gemini-3-pro-image-preview` (~$0.10) is the cheapest precision step in the lineup. Only upgrade once per loop. + +## Step 4: QC Pass #2 + +Same 5-bullet checklist. Document what changed between attempt 1 and attempt 2. + +## Step 5: Final Decision + +- **All 5 PASS:** Deliver with the file path + 1-line summary. +- **Some FAIL still:** Stop. Output: + - File path (the better of the two attempts) + - List of remaining failures + - **One** specific next move ("retry with `recraft_image` for SVG-grade text" / "use `text_card` from Remotion for the badge text" / "manual edit in Photoshop"). + +Never deliver a "best-effort" image without surfacing what's still wrong. + +--- + +## Output Format + +After every QC loop, the orchestrator should produce: + +``` +Image generation: +- Provider: : +- Output: +- QC status: PASS|PARTIAL|FAIL +- Checklist: + ✓ Composition match — + ✓ Required elements present — + ✗ Text fidelity — "PREMUM" should be "PREMIUM" → upgraded to gemini-3-pro-image-preview, fixed + ✓ Lighting + color — + ✓ Artifacts + anatomy — +- Auto-fix used: yes (Flash → 3 Pro for text fidelity) +- Cost: $0.14 (2 generations) +- Next step (if PARTIAL/FAIL): +``` + +--- + +## Integration with EP Gating + +This skill is **complementary** to the existing EP (Executive Producer) protocol used in cinematic / animation / video-clone pipelines. EP gating handles **multi-stage** quality control across an entire pipeline (proposal → script → scene → asset → edit → compose → publish). QC auto-fix handles **single-shot** images that bypass full EP. + +If you're already inside an EP-gated pipeline, the EP's existing review covers this. Don't double-run the QC loop. + +--- + +## Anti-Patterns + +1. **Retry forever** — hard cap is 2 generations. If a Flash-then-3-Pro-then-Flux-then-Recraft loop still fails, the prompt itself is broken. Report and stop. +2. **Skip QC because "it looked fine"** — every delivered image gets the 5-bullet check. Two bullets take 30 seconds; one missed broken word costs a re-render. +3. **Auto-fix without changing strategy** — re-running the same model with the same prompt produces the same defect. Always change something (model tier, prompt augmentation, family switch). +4. **Hallucinated PASS** — never tick PASS without evidence. "Looks good" is not evidence. +5. **Deliver before final QC** — even after auto-fix, run the checklist again. The fix may have introduced new issues. + +--- + +## When to skip this skill + +- Inside an EP-gated full pipeline (cinematic / animation / character-animation) — the EP already reviews. +- Throw-away test images for prompt iteration. +- Bulk variant generation (`num_variants > 1`) where the user explicitly wants raw outputs to pick from. +- Non-deliverable internal-only assets. + +--- + +## Relationship to OpenSwarm + +This pattern is ported from `VRSEN/OpenSwarm`'s `image_generation_agent/instructions.md` (commit 28e5fe38, 2026-05-08). OpenMontage adopts the **discipline** without the dependency — same QC ritual, applied to OpenMontage's broader 8-provider image lineup. diff --git a/tools/audio/qwen3_tts.py b/tools/audio/qwen3_tts.py new file mode 100644 index 000000000..981091457 --- /dev/null +++ b/tools/audio/qwen3_tts.py @@ -0,0 +1,312 @@ +"""Qwen3 TTS provider tool — local user-installed model via system Python. + +Runs in the user's global Python (3.14 / `/opt/homebrew/bin/python3.14`) +because `qwen-tts` is installed there (not in OpenMontage's .venv). +The tool calls the system Python via subprocess to run inference, +mirroring how PiperTTS shells out to the `piper` binary. + +Three inference modes from qwen_tts.Qwen3TTSModel: + - custom_voice : direct text→audio via stock voice + - voice_clone : reference audio sample drives the clone + - voice_design : natural-language voice description + +Default mode here is `custom_voice` (simplest path). + +User added 2026-05-07 per service-offering setup. +""" + +from __future__ import annotations + +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +SYSTEM_PYTHON = "/opt/homebrew/bin/python3.14" +DEFAULT_CHECKPOINT = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" + + +# Inline runner — keeps qwen3 inference code in one place, no separate file. +# This is a Python source string that the tool subprocess-executes via +# `python3.14 -c ` so it runs in the user's global env where qwen_tts +# is installed. +_RUNNER_SCRIPT = r""" +import sys +import json + +cfg = json.loads(sys.argv[1]) + +text = cfg["text"] +output_path = cfg["output_path"] +checkpoint = cfg.get("checkpoint", "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice") +mode = cfg.get("mode", "custom_voice") +speaker = cfg.get("speaker") +voice_description = cfg.get("voice_description") +ref_audio = cfg.get("reference_audio_path") +device = cfg.get("device", "cpu") +dtype = cfg.get("dtype", "float32") + +try: + from qwen_tts import Qwen3TTSModel +except ImportError as e: + print(json.dumps({"ok": False, "error": f"qwen_tts not importable: {e}"})) + sys.exit(1) + +try: + model = Qwen3TTSModel.from_pretrained(checkpoint, device=device, dtype=dtype) +except Exception as e: + print(json.dumps({"ok": False, "error": f"model load failed: {e}"})) + sys.exit(1) + +try: + if mode == "voice_design": + if not voice_description: + raise ValueError("voice_design mode needs `voice_description`") + wav = model.generate_voice_design(text=text, voice_description=voice_description) + elif mode == "voice_clone": + if not ref_audio: + raise ValueError("voice_clone mode needs `reference_audio_path`") + wav = model.generate_voice_clone(text=text, reference_audio=ref_audio) + else: + # custom_voice (default) + kwargs = {"text": text} + if speaker is not None: + kwargs["speaker"] = speaker + wav = model.generate_custom_voice(**kwargs) +except Exception as e: + print(json.dumps({"ok": False, "error": f"inference failed: {e}"})) + sys.exit(1) + +# `wav` is expected to be (audio_tensor, sample_rate) or similar — coerce to (samples, sr) +import torch, numpy as np, soundfile as sf +samples, sr = None, None +if isinstance(wav, tuple) and len(wav) == 2: + samples, sr = wav +elif hasattr(wav, "audio") and hasattr(wav, "sample_rate"): + samples, sr = wav.audio, wav.sample_rate +else: + print(json.dumps({"ok": False, "error": f"unexpected output shape: {type(wav)}"})) + sys.exit(1) + +if isinstance(samples, torch.Tensor): + samples = samples.detach().cpu().float().numpy() +samples = np.asarray(samples).squeeze() + +sf.write(output_path, samples, sr) +print(json.dumps({"ok": True, "output_path": output_path, "sample_rate": int(sr), "duration_s": len(samples)/sr})) +""" + + +class Qwen3TTS(BaseTool): + name = "qwen3_tts" + version = "0.1.0" + tier = ToolTier.VOICE + capability = "tts" + provider = "qwen" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC # voice design / clone is non-deterministic + runtime = ToolRuntime.LOCAL + + dependencies = [ + "binary:/opt/homebrew/bin/python3.14", + "package:qwen_tts (in system Python)", + "package:soundfile (in system Python)", + "package:torch (in system Python)", + ] + install_instructions = ( + "Qwen3 TTS runs in the user's GLOBAL Python (system /opt/homebrew/bin/python3.14),\n" + "not OpenMontage's .venv. To install on the system Python:\n" + " /opt/homebrew/bin/python3.14 -m pip install qwen-tts soundfile torch\n" + "First run downloads the checkpoint (~1-2 GB) from Hugging Face." + ) + agent_skills = ["text-to-speech"] + + capabilities = [ + "text_to_speech", + "voice_cloning", # generate_voice_clone + "voice_design", # generate_voice_design (natural-language description) + "offline_generation", + "multilingual", + ] + supports = { + "voice_cloning": True, + "multilingual": True, + "offline": True, + "native_audio": True, + } + best_for = [ + "user-preferred local TTS (memory: feedback_tts_qwen3_only.md)", + "voice clone from a 3-30 second reference clip", + "custom voice design via natural-language description", + "premium-quality narration without paid API", + ] + not_good_for = [ + "real-time low-latency (cold start ~30s for first inference)", + "machines without 4+ GB RAM (model is large)", + "GPU-less environments without `bfloat16` support (slow on CPU)", + ] + + input_schema = { + "type": "object", + "required": ["text"], + "properties": { + "text": {"type": "string", "description": "Text to synthesize (≤ ~500 chars per call recommended)"}, + "output_path": {"type": "string", "default": "qwen3_tts_output.wav"}, + "checkpoint": { + "type": "string", + "enum": [ + "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", + "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + ], + "default": DEFAULT_CHECKPOINT, + }, + "mode": { + "type": "string", + "enum": ["custom_voice", "voice_clone", "voice_design"], + "default": "custom_voice", + }, + "speaker": { + "type": "string", + "description": "(custom_voice mode) Stock speaker ID — see Qwen3TTSModel.get_supported_speakers()", + }, + "voice_description": { + "type": "string", + "description": "(voice_design mode) Natural-language voice description, e.g. 'warm female voice in her 20s, slightly breathy, conversational pace'", + }, + "reference_audio_path": { + "type": "string", + "description": "(voice_clone mode) Local path to 3-30s reference WAV/MP3 clip", + }, + "device": {"type": "string", "default": "cpu", "enum": ["cpu", "cuda:0", "mps"]}, + "dtype": {"type": "string", "default": "float32", "enum": ["float32", "bfloat16", "float16"]}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=4, ram_mb=4096, vram_mb=2048, disk_mb=2048, network_required=False + ) + retry_policy = RetryPolicy(max_retries=1, retryable_errors=[]) + idempotency_key_fields = ["text", "checkpoint", "mode", "speaker", "voice_description"] + side_effects = ["writes audio file to output_path", "downloads ~2GB checkpoint on first run"] + user_visible_verification = ["Listen to generated audio for naturalness + correct prosody"] + + # ── Status check ──────────────────────────────────────────────── + + def get_status(self) -> ToolStatus: + if not shutil.which(SYSTEM_PYTHON): + return ToolStatus.UNAVAILABLE + # Check qwen_tts module is importable in system python + try: + proc = subprocess.run( + [SYSTEM_PYTHON, "-c", "import qwen_tts; import soundfile; import torch; print('ok')"], + capture_output=True, + text=True, + timeout=15, + ) + if proc.returncode == 0 and "ok" in proc.stdout: + return ToolStatus.AVAILABLE + return ToolStatus.UNAVAILABLE + except Exception: + return ToolStatus.UNAVAILABLE + + # ── Cost ──────────────────────────────────────────────────────── + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + return 0.0 # local + + # ── Execute ───────────────────────────────────────────────────── + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if self.get_status() != ToolStatus.AVAILABLE: + return ToolResult( + success=False, + error="Qwen3 TTS not available. " + self.install_instructions, + ) + + start = time.time() + try: + result = self._generate(inputs) + except Exception as exc: + return ToolResult(success=False, error=f"Qwen3 TTS generation failed: {exc}") + + result.duration_seconds = round(time.time() - start, 2) + return result + + def _generate(self, inputs: dict[str, Any]) -> ToolResult: + import json + + output_path = Path(inputs.get("output_path", "qwen3_tts_output.wav")) + output_path.parent.mkdir(parents=True, exist_ok=True) + + cfg = { + "text": inputs["text"], + "output_path": str(output_path), + "checkpoint": inputs.get("checkpoint", DEFAULT_CHECKPOINT), + "mode": inputs.get("mode", "custom_voice"), + "speaker": inputs.get("speaker"), + "voice_description": inputs.get("voice_description"), + "reference_audio_path": inputs.get("reference_audio_path"), + "device": inputs.get("device", "cpu"), + "dtype": inputs.get("dtype", "float32"), + } + + proc = subprocess.run( + [SYSTEM_PYTHON, "-c", _RUNNER_SCRIPT, json.dumps(cfg)], + capture_output=True, + text=True, + timeout=600, # First inference can take 30-60s for model load + ) + + if proc.returncode != 0: + return ToolResult( + success=False, + error=f"Qwen3 subprocess failed (exit {proc.returncode}): {proc.stderr[-500:]}", + ) + + # Last line of stdout should be the JSON status + stdout = proc.stdout.strip().splitlines() + if not stdout: + return ToolResult(success=False, error="Qwen3 returned empty stdout") + try: + status = json.loads(stdout[-1]) + except json.JSONDecodeError: + return ToolResult(success=False, error=f"Qwen3 returned non-JSON: {stdout[-1][:200]}") + + if not status.get("ok"): + return ToolResult(success=False, error=f"Qwen3 inference: {status.get('error')}") + + if not output_path.exists(): + return ToolResult(success=False, error=f"Qwen3 output file missing: {output_path}") + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": cfg["checkpoint"], + "mode": cfg["mode"], + "text_length": len(inputs["text"]), + "output": str(output_path), + "format": "wav", + "sample_rate": status.get("sample_rate"), + "duration_s": status.get("duration_s"), + }, + artifacts=[str(output_path)], + model=cfg["checkpoint"], + ) diff --git a/tools/graphics/gemini_image.py b/tools/graphics/gemini_image.py new file mode 100644 index 000000000..6cf11d70a --- /dev/null +++ b/tools/graphics/gemini_image.py @@ -0,0 +1,223 @@ +"""Google Gemini image generation (gemini-2.5-flash-image / gemini-3-pro-image-preview). + +Ported from OpenSwarm 2026-05-08. Adds a third Gemini image provider to the +OpenMontage image-generation family alongside flux_image, openai_image, +recraft_image, grok_image, and local_diffusion. + +Default model: `gemini-2.5-flash-image` (fast, iteration-friendly). +Precision tier: `gemini-3-pro-image-preview` (text-heavy, complex compositions). +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class GeminiImage(BaseTool): + name = "gemini_image" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "image_generation" + provider = "google" + stability = ToolStability.BETA + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] # checked dynamically (google-genai) + install_instructions = ( + "Set GOOGLE_API_KEY to your Google AI Studio key.\n" + " pip install google-genai" + ) + agent_skills = ["flux-best-practices"] + + capabilities = [ + "generate_image", + "generate_illustration", + "text_to_image", + "high_precision_text_rendering", + ] + supports = { + "complex_instructions": True, + "text_in_image": True, + "multiple_outputs": True, + "broad_aspect_ratios": True, + } + best_for = [ + "fast iterative image workflows (gemini-2.5-flash-image)", + "text-heavy precision images (gemini-3-pro-image-preview)", + "complex product compositions with strict constraints", + "high-fidelity brand assets where prompt adherence matters", + ] + not_good_for = ["offline/private generation", "true SVG/vector output"] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string"}, + "model": { + "type": "string", + "enum": [ + "gemini-2.5-flash-image", + "gemini-3-pro-image-preview", + ], + "default": "gemini-2.5-flash-image", + }, + "aspect_ratio": { + "type": "string", + "enum": [ + "1:1", "2:3", "3:2", "3:4", "4:3", + "4:5", "5:4", "9:16", "16:9", "21:9", + ], + "default": "1:1", + }, + "num_variants": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 4, + }, + "output_path": {"type": "string"}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=200, network_required=True + ) + retry_policy = RetryPolicy( + max_retries=2, retryable_errors=["rate_limit", "timeout"] + ) + idempotency_key_fields = ["prompt", "aspect_ratio", "model"] + side_effects = [ + "writes image file to output_path", + "calls Google Gemini API", + ] + user_visible_verification = [ + "Inspect generated image for relevance, prompt adherence, and text accuracy" + ] + + def get_status(self) -> ToolStatus: + if os.environ.get("GOOGLE_API_KEY"): + return ToolStatus.AVAILABLE + return ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + # Gemini 2.5 Flash Image: ~$0.039/image (cheap, fast) + # Gemini 3 Pro Image Preview: ~$0.10/image (precision tier) + # Both per output, scales with num_variants. + model = inputs.get("model", "gemini-2.5-flash-image") + n = inputs.get("num_variants", 1) + per_image = 0.10 if model == "gemini-3-pro-image-preview" else 0.039 + return per_image * n + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if not os.environ.get("GOOGLE_API_KEY"): + return ToolResult( + success=False, + error="GOOGLE_API_KEY not set. " + self.install_instructions, + ) + + try: + from google import genai + from google.genai.types import GenerateContentConfig, ImageConfig + except ImportError: + return ToolResult( + success=False, + error=( + "google-genai not installed. Run: pip install google-genai" + ), + ) + + start = time.time() + client = genai.Client(api_key=os.environ["GOOGLE_API_KEY"]) + model = inputs.get("model", "gemini-2.5-flash-image") + prompt = inputs["prompt"] + aspect_ratio = inputs.get("aspect_ratio", "1:1") + n = inputs.get("num_variants", 1) + output_path_template = inputs.get("output_path", "generated_image.png") + + artifacts: list[str] = [] + try: + for idx in range(n): + response = client.models.generate_content( + model=model, + contents=prompt, + config=GenerateContentConfig( + image_config=ImageConfig(aspect_ratio=aspect_ratio), + ), + ) + + image_bytes = self._extract_image_bytes(response) + if image_bytes is None: + continue + + if n > 1: + base = Path(output_path_template) + out_path = base.with_name(f"{base.stem}_v{idx + 1}{base.suffix}") + else: + out_path = Path(output_path_template) + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(image_bytes) + artifacts.append(str(out_path)) + + except Exception as exc: + return ToolResult( + success=False, + error=f"Gemini image generation failed: {exc}", + ) + + if not artifacts: + return ToolResult( + success=False, + error="Gemini returned no image data (safety filter or empty response).", + ) + + return ToolResult( + success=True, + data={ + "provider": "google", + "model": model, + "prompt": prompt, + "aspect_ratio": aspect_ratio, + "outputs": artifacts, + }, + artifacts=artifacts, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + model=model, + ) + + @staticmethod + def _extract_image_bytes(response: Any) -> bytes | None: + """Pull raw image bytes out of a Gemini generate_content response. + + Gemini returns multimodal Parts; the image is in `inline_data.data` as + bytes, content type `image/png` or `image/jpeg`. + """ + try: + for candidate in getattr(response, "candidates", []) or []: + for part in getattr(candidate.content, "parts", []) or []: + inline = getattr(part, "inline_data", None) + if inline and getattr(inline, "data", None): + return inline.data + except Exception: + return None + return None diff --git a/tools/graphics/kie_gpt_image.py b/tools/graphics/kie_gpt_image.py new file mode 100644 index 000000000..826e7fcaa --- /dev/null +++ b/tools/graphics/kie_gpt_image.py @@ -0,0 +1,197 @@ +"""GPT Image 2 (and 4o-image) via KIE.AI. + +Best for: logo wordmarks, packaging mockups with readable label text, +typography-heavy product shots — OpenAI image models have the strongest +text-rendering in the current commercial fleet. + +Reference: ~/.claude/projects/-Users-abalioglu/memory/reference_kieai_models.md +- Pattern A: `openai/gpt-image-2` via /jobs/createTask +- Pattern B: `4o Image` via dedicated /gpt4o-image/generate (fallback) + +User added 2026-05-07 alongside Nano Banana 2 + Qwen3 TTS. +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) +from lib import kie_client + + +class KIEGPTImage(BaseTool): + name = "kie_gpt_image" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "image_generation" + provider = "kie:openai_gpt_image" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = "Set KIE_AI_API_KEY (https://kie.ai)." + agent_skills = ["flux-best-practices"] + + capabilities = ["text_to_image", "text_in_image", "image_to_image"] + supports = { + "text_to_image": True, + "text_in_image": True, # the moat — readable typography + "image_to_image": True, + "high_resolution": True, # 1024×1792 / 1792×1024 + } + best_for = [ + "logo wordmarks, brand marks with readable text", + "packaging mockups with label text (bottle, box, tube)", + "infographic / data visualization with embedded labels", + "any product shot where text MUST be legible", + ] + not_good_for = [ + "best price-per-image (Nano Banana is cheaper for plain product/character)", + "extreme photorealism (Imagen 4 / Flux often score higher on pure photoreal)", + ] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string", "description": "1-20,000 characters"}, + "model": { + "type": "string", + "enum": [ + "gpt-image-2-text-to-image", + "gpt-image-2-image-to-image", + ], + "default": "gpt-image-2-text-to-image", + "description": ( + "text-to-image: prompt only. " + "image-to-image: prompt + input_urls (max 16). " + "If reference_image_urls is provided, model auto-switches to image-to-image." + ), + }, + "aspect_ratio": { + "type": "string", + "enum": ["auto", "1:1", "9:16", "16:9", "4:3", "3:4"], + "default": "auto", + }, + "resolution": { + "type": "string", + "enum": ["1K", "2K", "4K"], + "default": "1K", + "description": "1:1 cannot use 4K. auto aspect_ratio limited to 1K.", + }, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 16, + "description": "Reference image URLs/paths (local auto-uploaded). Triggers image-to-image mode.", + }, + "output_path": {"type": "string", "default": "gpt_image_output.png"}, + }, + } + + resource_profile = ResourceProfile(cpu_cores=1, ram_mb=128, vram_mb=0, disk_mb=10, network_required=True) + retry_policy = RetryPolicy(max_retries=2, retryable_errors=["timeout", "5xx"]) + idempotency_key_fields = ["prompt", "model", "size", "quality", "reference_image_urls"] + side_effects = ["calls KIE.AI", "writes image file to output_path"] + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if kie_client.is_configured() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + # KIE pricing scales with resolution: ~$0.04 (1K), ~$0.10 (2K), ~$0.20 (4K) + # Adjust per official KIE catalog if these drift. + resolution = inputs.get("resolution", "1K") + return {"1K": 0.04, "2K": 0.10, "4K": 0.20}.get(resolution, 0.04) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if self.get_status() != ToolStatus.AVAILABLE: + return ToolResult(success=False, error="KIE_AI_API_KEY not set. " + self.install_instructions) + + start = time.time() + try: + output_path = Path(inputs.get("output_path", "gpt_image_output.png")) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Resolve any local reference images + ref_urls = [] + for ref in inputs.get("reference_image_urls", []) or []: + ref_urls.append(kie_client.maybe_upload(ref)) + + # Auto-pick model based on whether refs are provided + model = inputs.get("model") + if not model: + model = "gpt-image-2-image-to-image" if ref_urls else "gpt-image-2-text-to-image" + elif ref_urls and model == "gpt-image-2-text-to-image": + # User asked for text-to-image but provided refs → switch + model = "gpt-image-2-image-to-image" + + payload: dict[str, Any] = { + "prompt": inputs["prompt"], + "aspect_ratio": inputs.get("aspect_ratio", "auto"), + "resolution": inputs.get("resolution", "1K"), + } + + # Constraint: 1:1 cannot use 4K + if payload["aspect_ratio"] == "1:1" and payload["resolution"] == "4K": + payload["resolution"] = "2K" + + # Constraint: auto aspect_ratio limited to 1K + if payload["aspect_ratio"] == "auto" and payload["resolution"] != "1K": + payload["resolution"] = "1K" + + # image-to-image needs input_urls + if model == "gpt-image-2-image-to-image": + if not ref_urls: + return ToolResult( + success=False, + error="gpt-image-2-image-to-image requires reference_image_urls (input_urls)", + ) + payload["input_urls"] = ref_urls[:16] # max 16 per spec + + record = kie_client.run_unified(model, payload, max_wait_s=300) + urls = kie_client.extract_result_urls(record) + + if not urls: + return ToolResult( + success=False, + error=f"GPT Image returned no result URLs (model={model}): {record}", + ) + + kie_client.download_to(urls[0], output_path) + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": model, + "prompt_length": len(inputs["prompt"]), + "aspect_ratio": payload["aspect_ratio"], + "resolution": payload["resolution"], + "output": str(output_path), + "all_urls": urls, + "format": "png", + }, + artifacts=[str(output_path)], + model=model, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + ) + except kie_client.KIEError as exc: + return ToolResult(success=False, error=f"KIE GPT Image: {exc}", duration_seconds=round(time.time() - start, 2)) + except Exception as exc: + return ToolResult(success=False, error=f"KIE GPT Image unexpected: {exc}", duration_seconds=round(time.time() - start, 2)) diff --git a/tools/graphics/kie_nano_banana.py b/tools/graphics/kie_nano_banana.py new file mode 100644 index 000000000..65b1454fb --- /dev/null +++ b/tools/graphics/kie_nano_banana.py @@ -0,0 +1,150 @@ +"""Nano Banana 2 image generation via KIE.AI. + +Google Gemini 2.5 Flash Image — multimodal, accepts up to 14 reference images +(Pseudo-Soul ID pattern for character consistency across shots). + +Best for: product photography, character master refs, multi-ref consistency. +Reference grammar: ~/.claude/skills/model-cheatsheet.md → Nano Banana section. + +User added 2026-05-07 alongside Qwen3 TTS. +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) +from lib import kie_client + + +class KIENanoBanana(BaseTool): + name = "kie_nano_banana" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "image_generation" + provider = "kie:google_nano_banana" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = "Set KIE_AI_API_KEY (https://kie.ai)." + agent_skills = ["ai-video-gen", "flux-best-practices"] + + capabilities = ["text_to_image", "multi_reference_to_image"] + supports = { + "text_to_image": True, + "image_to_image": True, + "multi_reference": True, # up to 14 ref images (Pseudo-Soul ID) + "character_consistency": True, + } + best_for = [ + "character master ref + 9-angle production", + "product hero photography (clean glass, soft window light)", + "scene continuity across multi-shot videos (re-use same hero ref)", + ] + not_good_for = [ + "logos / packaging text rendering — use kie_gpt_image instead", + "pure typography or graphic design output", + ] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string"}, + "model": { + "type": "string", + "enum": ["google/nano-banana", "nano-banana-2"], + "default": "nano-banana-2", + }, + "aspect_ratio": { + "type": "string", + "enum": ["1:1", "9:16", "16:9", "3:4", "4:3"], + "default": "1:1", + }, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Up to 14 reference URLs/local-paths. Local paths are auto-uploaded.", + "maxItems": 14, + }, + "n": {"type": "integer", "default": 1, "minimum": 1, "maximum": 4}, + "output_path": {"type": "string", "default": "nano_banana_output.png"}, + }, + } + + resource_profile = ResourceProfile(cpu_cores=1, ram_mb=128, vram_mb=0, disk_mb=10, network_required=True) + retry_policy = RetryPolicy(max_retries=2, retryable_errors=["timeout", "5xx"]) + idempotency_key_fields = ["prompt", "model", "aspect_ratio", "reference_image_urls"] + side_effects = ["calls KIE.AI", "writes image file to output_path"] + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if kie_client.is_configured() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + return 0.04 # ~$0.04 per image (KIE catalog) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if self.get_status() != ToolStatus.AVAILABLE: + return ToolResult(success=False, error="KIE_AI_API_KEY not set. " + self.install_instructions) + + start = time.time() + try: + output_path = Path(inputs.get("output_path", "nano_banana_output.png")) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Resolve any local reference images + ref_urls = [] + for ref in inputs.get("reference_image_urls", []) or []: + ref_urls.append(kie_client.maybe_upload(ref)) + + payload: dict[str, Any] = { + "prompt": inputs["prompt"], + "aspect_ratio": inputs.get("aspect_ratio", "1:1"), + "n": int(inputs.get("n", 1)), + } + if ref_urls: + payload["reference_image_urls"] = ref_urls + + model = inputs.get("model", "nano-banana-2") + record = kie_client.run_unified(model, payload, max_wait_s=300) + urls = kie_client.extract_result_urls(record) + if not urls: + return ToolResult(success=False, error=f"Nano Banana returned no result URLs: {record}") + + kie_client.download_to(urls[0], output_path) + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": model, + "prompt_length": len(inputs["prompt"]), + "output": str(output_path), + "all_urls": urls, + "format": "png", + }, + artifacts=[str(output_path)], + model=model, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + ) + except kie_client.KIEError as exc: + return ToolResult(success=False, error=f"KIE Nano Banana: {exc}", duration_seconds=round(time.time() - start, 2)) + except Exception as exc: + return ToolResult(success=False, error=f"KIE Nano Banana unexpected: {exc}", duration_seconds=round(time.time() - start, 2)) diff --git a/tools/video/kie_kling.py b/tools/video/kie_kling.py new file mode 100644 index 000000000..5e2bf22ca --- /dev/null +++ b/tools/video/kie_kling.py @@ -0,0 +1,205 @@ +"""Kling 3.0 / 2.6 video generation via KIE.AI. + +Alternative gateway to the existing fal.ai-based `kling_video` tool. +Uses KIE_AI_API_KEY instead of FAL_KEY. + +Reference: ~/.claude/projects/-Users-abalioglu/memory/reference_kieai_models.md +- kling-2.6/image-to-video — duration "5"/"10", sound true/false +- kling-3.0/video — duration 3-15s, multi_shots, multi_prompt + +Best for cinematic B-roll, fluid camera, mid-tier UGC product replacement. +Cheaper than Seedance (~$0.08/s vs $0.24-0.30/s). + +User added 2026-05-07. +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) +from lib import kie_client + + +class KIEKling(BaseTool): + name = "kie_kling" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "video_generation" + provider = "kie:kling" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = "Set KIE_AI_API_KEY (https://kie.ai)." + agent_skills = ["ai-video-gen"] + + capabilities = ["text_to_video", "image_to_video"] + supports = { + "text_to_video": True, + "image_to_video": True, + "native_audio": True, + "cinematic_quality": True, + "multi_shot": True, # kling-3.0 only + "multi_prompt": True, # kling-3.0 only + } + best_for = [ + "cost-effective UGC b-roll and product replacement (~$0.08/s)", + "fluid camera motion, smooth dolly/push-in shots", + "mid-tier cinematic where Seedance is overkill", + ] + not_good_for = [ + "lip-sync from quoted dialogue (use Seedance or talking-head pipeline)", + "complex multi-shot identity persistence (Seedance is stronger here)", + "anamorphic / film grain aesthetics (forbidden words in Kling — see model-cheatsheet.md)", + ] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string", "description": "≤150 words. Avoid: anamorphic, film grain, halation."}, + "model": { + "type": "string", + "enum": [ + "kling-3.0/video", + "kling-3.0/image-to-video", + "kling-2.6/image-to-video", + "kling-2.6/text-to-video", + ], + "default": "kling-3.0/video", + }, + "duration": { + "type": ["integer", "string"], + "default": 5, + "description": "kling-3.0: integer 3-15s. kling-2.6: string '5' or '10'.", + }, + "aspect_ratio": { + "type": "string", + "enum": ["9:16", "16:9", "1:1"], + "default": "9:16", + }, + "image_url": { + "type": "string", + "description": "i2v start frame (URL or local — local auto-uploaded). Required for image-to-video models.", + }, + "sound": {"type": "boolean", "default": True, "description": "kling-2.6 only — sound:true/false"}, + "multi_shots": { + "type": "boolean", + "default": False, + "description": "kling-3.0 only — enable multi-shot interpretation", + }, + "multi_prompt": { + "type": "array", + "items": {"type": "string"}, + "description": "kling-3.0 only — sequential prompts for multi-shot beats", + }, + "negative_prompt": {"type": "string"}, + "seed": {"type": "integer"}, + "output_path": {"type": "string", "default": "kling_output.mp4"}, + }, + } + + resource_profile = ResourceProfile(cpu_cores=1, ram_mb=128, vram_mb=0, disk_mb=200, network_required=True) + retry_policy = RetryPolicy(max_retries=1, retryable_errors=["timeout", "5xx"]) + idempotency_key_fields = [ + "prompt", "model", "duration", "aspect_ratio", "image_url", + "multi_shots", "multi_prompt", "negative_prompt", "seed", + ] + side_effects = ["calls KIE.AI", "writes video file to output_path"] + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if kie_client.is_configured() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + # KIE Kling pricing ~$0.08/s for 3.0, slightly less for 2.6 + duration = inputs.get("duration", 5) + try: + duration_int = int(str(duration)) + except ValueError: + duration_int = 5 + rate = 0.08 if "3.0" in inputs.get("model", "kling-3.0/video") else 0.07 + return round(rate * duration_int, 3) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if self.get_status() != ToolStatus.AVAILABLE: + return ToolResult(success=False, error="KIE_AI_API_KEY not set. " + self.install_instructions) + + start = time.time() + try: + output_path = Path(inputs.get("output_path", "kling_output.mp4")) + output_path.parent.mkdir(parents=True, exist_ok=True) + + model = inputs.get("model", "kling-3.0/video") + + payload: dict[str, Any] = { + "prompt": inputs["prompt"], + "duration": inputs.get("duration", 5), + "aspect_ratio": inputs.get("aspect_ratio", "9:16"), + } + if "negative_prompt" in inputs: + payload["negative_prompt"] = inputs["negative_prompt"] + if "seed" in inputs: + payload["seed"] = int(inputs["seed"]) + + # Image-to-video models need image_url + if "image-to-video" in model or "i2v" in model: + if not inputs.get("image_url"): + return ToolResult(success=False, error=f"{model} requires image_url") + payload["image_url"] = kie_client.maybe_upload(inputs["image_url"]) + + # kling-2.6-specific + if "2.6" in model: + payload["sound"] = bool(inputs.get("sound", True)) + # 2.6 wants string duration + payload["duration"] = str(payload["duration"]) + + # kling-3.0-specific + if "3.0" in model: + if inputs.get("multi_shots"): + payload["multi_shots"] = True + if inputs.get("multi_prompt"): + payload["multi_prompt"] = inputs["multi_prompt"] + + record = kie_client.run_unified(model, payload, max_wait_s=900) + urls = kie_client.extract_result_urls(record) + if not urls: + return ToolResult(success=False, error=f"Kling returned no result URLs: {record}") + + kie_client.download_to(urls[0], output_path) + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": model, + "duration": payload["duration"], + "aspect_ratio": payload["aspect_ratio"], + "output": str(output_path), + "all_urls": urls, + "format": "mp4", + }, + artifacts=[str(output_path)], + model=model, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + ) + except kie_client.KIEError as exc: + return ToolResult(success=False, error=f"KIE Kling: {exc}", duration_seconds=round(time.time() - start, 2)) + except Exception as exc: + return ToolResult(success=False, error=f"KIE Kling unexpected: {exc}", duration_seconds=round(time.time() - start, 2)) diff --git a/tools/video/kie_seedance.py b/tools/video/kie_seedance.py new file mode 100644 index 000000000..14b355303 --- /dev/null +++ b/tools/video/kie_seedance.py @@ -0,0 +1,191 @@ +"""Seedance 2 / Seedance 2 Fast video generation via KIE.AI. + +Alternative gateway to the existing fal.ai-based `seedance_video` tool. +Uses KIE_AI_API_KEY instead of FAL_KEY. + +Reference: ~/.claude/projects/-Users-abalioglu/memory/reference_kieai_models.md +- bytedance/seedance-2 — duration 4-15s, aspect 9:16/16:9/1:1/adaptive, + first_frame_url (i2v), reference_image_urls (max 9), generate_audio, + web_search (must be false) +- bytedance/seedance-2-fast — same params, faster/cheaper, for tests/sample renders + +User added 2026-05-07. +""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) +from lib import kie_client + + +class KIESeedance(BaseTool): + name = "kie_seedance" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "video_generation" + provider = "kie:bytedance_seedance" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = "Set KIE_AI_API_KEY (https://kie.ai)." + agent_skills = ["seedance-2-0", "ai-video-gen"] + + capabilities = ["text_to_video", "image_to_video", "reference_to_video"] + supports = { + "text_to_video": True, + "image_to_video": True, + "reference_to_video": True, + "native_audio": True, + "multi_shot": True, + "lip_sync_from_quoted_dialogue": True, + "character_identity_consistency": True, + } + best_for = [ + "premium cinematic clips with multi-shot identity persistence", + "trailer/teaser/hype-edit beats", + "lip-sync from prompt-quoted dialogue (`Character says: \"...\"` pattern)", + "reference-conditioned generation — up to 9 ref images", + ] + not_good_for = [ + "cheap b-roll (use kling_video or pexels_video for cost)", + "4+ simultaneous actions in one shot (split to multi-shot)", + "readable text/logos inside the clip (handle text in Remotion overlay)", + ] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string", "description": "200-400 words for hero shots; 80-150 for inserts"}, + "model": { + "type": "string", + "enum": ["bytedance/seedance-2", "bytedance/seedance-2-fast"], + "default": "bytedance/seedance-2-fast", + "description": "Use fast for samples/previews/cost-capped; standard for hero/multi-shot/camera-heavy.", + }, + "duration": {"type": "integer", "default": 5, "minimum": 4, "maximum": 15}, + "aspect_ratio": { + "type": "string", + "enum": ["9:16", "16:9", "1:1", "4:3", "3:4", "21:9", "adaptive"], + "default": "9:16", + }, + "resolution": {"type": "string", "enum": ["480p", "720p"], "default": "720p"}, + "first_frame_url": { + "type": "string", + "description": "i2v starting frame (URL or local path — local auto-uploaded). Mutually exclusive with reference_image_urls.", + }, + "last_frame_url": {"type": "string", "description": "Optional last-frame anchor."}, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 9, + "description": "Multi-ref mode (up to 9). Mutually exclusive with first_frame_url.", + }, + "generate_audio": {"type": "boolean", "default": True}, + "seed": {"type": "integer"}, + "output_path": {"type": "string", "default": "seedance_output.mp4"}, + }, + } + + resource_profile = ResourceProfile(cpu_cores=1, ram_mb=128, vram_mb=0, disk_mb=200, network_required=True) + retry_policy = RetryPolicy(max_retries=1, retryable_errors=["timeout", "5xx"]) + idempotency_key_fields = [ + "prompt", "model", "duration", "aspect_ratio", "resolution", + "first_frame_url", "last_frame_url", "reference_image_urls", "seed", + ] + side_effects = ["calls KIE.AI", "writes video file to output_path"] + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if kie_client.is_configured() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + # KIE pricing approximations (2026): + # seedance-2 standard: ~$0.30/s + # seedance-2-fast: ~$0.24/s + model = inputs.get("model", "bytedance/seedance-2-fast") + duration = float(inputs.get("duration", 5)) + rate = 0.30 if model == "bytedance/seedance-2" else 0.24 + return round(rate * duration, 3) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + if self.get_status() != ToolStatus.AVAILABLE: + return ToolResult(success=False, error="KIE_AI_API_KEY not set. " + self.install_instructions) + + start = time.time() + try: + output_path = Path(inputs.get("output_path", "seedance_output.mp4")) + output_path.parent.mkdir(parents=True, exist_ok=True) + + payload: dict[str, Any] = { + "prompt": inputs["prompt"], + "duration": int(inputs.get("duration", 5)), + "aspect_ratio": inputs.get("aspect_ratio", "9:16"), + "resolution": inputs.get("resolution", "720p"), + "generate_audio": bool(inputs.get("generate_audio", True)), + "web_search": False, # mandatory for KIE — see memory/reference_kieai_models.md + } + if "seed" in inputs: + payload["seed"] = int(inputs["seed"]) + + # Resolve i2v / multi-ref (mutually exclusive) + first_frame = inputs.get("first_frame_url") + ref_imgs = inputs.get("reference_image_urls") or [] + if first_frame and ref_imgs: + return ToolResult( + success=False, + error="seedance: first_frame_url and reference_image_urls are mutually exclusive — pick one", + ) + if first_frame: + payload["first_frame_url"] = kie_client.maybe_upload(first_frame) + if inputs.get("last_frame_url"): + payload["last_frame_url"] = kie_client.maybe_upload(inputs["last_frame_url"]) + elif ref_imgs: + payload["reference_image_urls"] = [kie_client.maybe_upload(u) for u in ref_imgs[:9]] + + model = inputs.get("model", "bytedance/seedance-2-fast") + record = kie_client.run_unified(model, payload, max_wait_s=900) # 15min for full quality + urls = kie_client.extract_result_urls(record) + if not urls: + return ToolResult(success=False, error=f"Seedance returned no result URLs: {record}") + + kie_client.download_to(urls[0], output_path) + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": model, + "duration_s": payload["duration"], + "aspect_ratio": payload["aspect_ratio"], + "resolution": payload["resolution"], + "output": str(output_path), + "all_urls": urls, + "format": "mp4", + }, + artifacts=[str(output_path)], + model=model, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + ) + except kie_client.KIEError as exc: + return ToolResult(success=False, error=f"KIE Seedance: {exc}", duration_seconds=round(time.time() - start, 2)) + except Exception as exc: + return ToolResult(success=False, error=f"KIE Seedance unexpected: {exc}", duration_seconds=round(time.time() - start, 2))