Merge pull request #393 from howardbaik/feat-prompt-attachments

Support multi-modal inputs (images, PDFs) in `Validate.prompt()`

Author: rich-iannone

pointblank/_interrogation.py

@@ -2821,6 +2821,7 @@ def interrogate_prompt(
         llm_model = ai_config["llm_model"]
         batch_size = ai_config.get("batch_size", 1000)
         max_concurrent = ai_config.get("max_concurrent", 3)
+        attachments = ai_config.get("attachments", [])
 
         # Set up LLM configuration (api_key will be loaded from environment)
         llm_config = _LLMConfig(
@@ -2852,7 +2853,7 @@ def interrogate_prompt(
         prompt_builder = _PromptBuilder(prompt)
 
         # Create AI validation engine
-        engine = _AIValidationEngine(llm_config)
+        engine = _AIValidationEngine(llm_config, attachments=attachments)
 
         # Run AI validation synchronously (chatlas is synchronous)
         batch_results = engine.validate_batches(

pointblank/_utils_ai.py

@@ -3,13 +3,18 @@
 import hashlib
 import json
 import logging
+import pathlib
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
 
 import narwhals as nw
 
 from pointblank._constants import MODEL_PROVIDERS
 
+_IMAGE_EXTS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp"})
+_PDF_EXTS = frozenset({".pdf"})
+_SUPPORTED_ATTACHMENT_EXTS = _IMAGE_EXTS | _PDF_EXTS
+
 logger = logging.getLogger(__name__)
 
 
@@ -93,7 +98,9 @@ def _create_chat_instance(
   {"index": 0, "result": true},
   {"index": 1, "result": false},
   {"index": 2, "result": true}
-]"""
+]
+
+If reference attachments (images or PDFs) are provided alongside the data, use them as context when evaluating each row."""
 
     # Create httpx client with SSL verification settings
     try:
@@ -213,6 +220,71 @@ def _create_chat_instance(
     return chat
 
 
+def _prepare_attachments(attachments: Optional[List[Any]]) -> List[Any]:
+    """
+    Coerce a heterogeneous list of attachment specs into chatlas Content objects.
+
+    Accepts:
+    - ``str`` or ``pathlib.Path``: auto-converted to ``content_image_*`` or ``content_pdf_*``
+      based on extension. URLs (``http://``/``https://``) use the URL variants.
+    - Pre-built chatlas ``Content`` objects: passed through untouched.
+
+    Raises ``ValueError`` for unsupported extensions and ``ImportError`` if chatlas
+    is not installed.
+    """
+    if not attachments:
+        return []
+
+    if not isinstance(attachments, (list, tuple)):
+        raise TypeError(f"attachments must be a list or tuple, got {type(attachments).__name__}")
+
+    try:
+        from chatlas import (
+            content_image_file,
+            content_image_url,
+            content_pdf_file,
+            content_pdf_url,
+        )
+    except ImportError:  # pragma: no cover
+        raise ImportError(
+            "The `chatlas` package is required for attachments support. "
+            "Please install it using `pip install chatlas`."
+        )
+
+    prepared: List[Any] = []
+    for item in attachments:
+        # If it's a string or path-like, coerce based on extension. Otherwise
+        # assume it's an already-built chatlas Content object and pass through.
+        if isinstance(item, (str, pathlib.PurePath)):
+            path_str = str(item)
+            # Strip query string before checking extension for URLs.
+            ext = pathlib.PurePosixPath(path_str.split("?", 1)[0]).suffix.lower()
+            if ext not in _SUPPORTED_ATTACHMENT_EXTS:
+                raise ValueError(
+                    f"Unsupported attachment extension {ext!r} for {path_str!r}. "
+                    f"Supported extensions: {sorted(_SUPPORTED_ATTACHMENT_EXTS)}"
+                )
+            is_url = path_str.startswith(("http://", "https://"))
+            if ext in _IMAGE_EXTS:
+                # content_image_file emits MissingResizeWarning when no resize
+                # is given; pass "low" explicitly (its implicit default) to
+                # suppress it. content_image_url has no resize parameter.
+                # Users wanting higher fidelity should pre-build the Content
+                # with their preferred resize and pass it in directly (the
+                # pass-through branch below handles that).
+                prepared.append(
+                    content_image_url(path_str)
+                    if is_url
+                    else content_image_file(path_str, resize="low")
+                )
+            else:
+                prepared.append(content_pdf_url(path_str) if is_url else content_pdf_file(path_str))
+        else:
+            prepared.append(item)
+
+    return prepared
+
+
 # ============================================================================
 # Data Batching and Optimization
 # ============================================================================
@@ -767,16 +839,24 @@ def combine_batch_results(
 class _AIValidationEngine:
     """Main engine for AI-powered validation using chatlas."""
 
-    def __init__(self, llm_config: _LLMConfig):
+    def __init__(
+        self,
+        llm_config: _LLMConfig,
+        attachments: Optional[List[Any]] = None,
+    ):
         """
         Initialize the AI validation engine.
 
         Parameters
         ----------
         llm_config
             Configuration for the LLM provider.
+        attachments
+            Optional list of chatlas ``Content`` objects sent as global context
+            alongside every batch's text prompt.
         """
         self.llm_config = llm_config
+        self.attachments: List[Any] = list(attachments) if attachments else []
         self.chat = _create_chat_instance(
             provider=llm_config.provider,
             model_name=llm_config.model,
@@ -823,8 +903,9 @@ def validate_batch(batch: Dict[str, Any]) -> List[Dict[str, Any]]:
                 logger.debug(prompt)
                 logger.debug("--- PROMPT END ---")
 
-                # Get response from LLM using chatlas (synchronous)
-                response = str(self.chat.chat(prompt, stream=False, echo="none"))
+                # Get response from LLM using chatlas (synchronous). Attachments
+                # ride along as global context content blocks for every batch.
+                response = str(self.chat.chat(prompt, *self.attachments, stream=False, echo="none"))
 
                 # Debug: Log the raw LLM response
                 logger.debug(f"📥 LLM Response for batch {batch['batch_id']}:")
@@ -890,8 +971,9 @@ def validate_single_batch(
             # Build the prompt for this batch
             prompt = prompt_builder.build_prompt(batch["data"])
 
-            # Get response from LLM using chatlas (synchronous)
-            response = str(self.chat.chat(prompt, stream=False, echo="none"))
+            # Get response from LLM using chatlas (synchronous). Attachments
+            # ride along as global context content blocks for every batch.
+            response = str(self.chat.chat(prompt, *self.attachments, stream=False, echo="none"))
 
             # Parse the response
             parser = _ValidationResponseParser(total_rows=1000)  # This will be set properly

pointblank/validate.py

@@ -10881,6 +10881,7 @@ def prompt(
         prompt: str,
         model: str,
         columns_subset: str | list[str] | None = None,
+        attachments: list | None = None,
         batch_size: int = 1000,
         max_concurrent: int = 3,
         pre: Callable | None = None,
@@ -10926,6 +10927,15 @@ def prompt(
             A single column or list of columns to include in the validation. If `None`, all columns
             will be included. Specifying fewer columns can improve performance and reduce API costs
             so try to include only the columns necessary for the validation.
+        attachments
+            An optional list of reference files (images or PDFs) to attach as global context for
+            every batch. Each item can be a local file path, a URL (`http://` / `https://`), a
+            `pathlib.Path`, or a pre-built chatlas `Content` object. Supported extensions are
+            `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, and `.pdf`. The attachments apply to the whole
+            validation step, not per-row, so they are well-suited for things like brand guides,
+            schema diagrams, or sample documents the LLM should consult when scoring each row.
+            **Cost note**: attachments are re-sent on every batch — see the *Multi-modal
+            attachments* section below for cost-management tips.
         model
             The model to be used. This should be in the form of `provider:model` (e.g.,
             `"anthropic:claude-opus-4-6"`). Supported providers are `"anthropic"`, `"openai"`,
@@ -11094,6 +11104,39 @@ def prompt(
         - "Describe the quality of each row" (asks for description, not validation)
         - "How would you improve this data?" (asks for suggestions, not pass/fail)
 
+        Multi-modal Attachments
+        -----------------------
+        Use `attachments=` to give the LLM a reference image or PDF that applies to every row. The
+        attachment is sent as global context, while each row's values are still serialized as JSON
+        and validated against your `prompt=`. Useful patterns:
+
+        - validating descriptions against a brand-style image: `attachments=["brand_guide.pdf"]`
+        - cross-referencing rows with a schema diagram: `attachments=["schema.png"]`
+        - matching free-text fields to a sample document: `attachments=["sample_invoice.pdf"]`
+
+        Accepted values per list item:
+
+        - local path strings or `pathlib.Path` objects (e.g., `"docs/diagram.png"`)
+        - URLs (e.g., `"https://example.com/diagram.png"`)
+        - pre-built chatlas `Content` objects (e.g., `chatlas.content_image_plot()`)
+
+        Supported extensions: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.pdf`.
+
+        Note: local image paths auto-coerced through `attachments=` use `resize="low"` (chatlas's
+        default downscale to 512x512) to keep token costs predictable. For higher fidelity,
+        pre-build the content yourself with `chatlas.content_image_file(path, resize="high")`
+        (or `"none"`) and pass that object inside `attachments=`.
+
+        **Cost / batching note**: attachments are re-sent on *every* batch (one batch = one LLM API
+        call). For a table that requires N batches, each attachment's input tokens are billed N
+        times. To control costs:
+
+        - keep attachments small (downscale images, crop PDFs to the relevant page)
+        - use `columns_subset=` aggressively to maximize row-signature memoization (fewer unique
+          rows means fewer batches)
+        - raise `batch_size=` when the combined system prompt, attachment, and row JSON fit
+          comfortably under the model's context window
+
         Performance Considerations
         --------------------------
         AI validation is significantly slower than traditional validation methods due to API calls
@@ -11215,6 +11258,28 @@ def prompt(
         which exceeds all threshold levels. The validation will trigger the specified error action
         since the failure rate (40%) is above the error threshold (20%). The AI can recognize
         various phone number formats and determine whether they include area codes.
+
+        **Multi-modal example with `attachments=`:**
+
+        Suppose you have a table of product descriptions and a brand-style PDF that describes the
+        approved tone and vocabulary. Pass the PDF as a global attachment so the LLM can compare
+        each description against it.
+
+        ```python
+        validation = (
+            pb.Validate(data=products)
+            .prompt(
+                prompt="Each product description must match the tone and vocabulary in the brand guide.",
+                columns_subset=["description"],
+                attachments=["docs/brand_guide.pdf"],
+                model="anthropic:claude-opus-4-6",
+            )
+            .interrogate()
+        )
+        ```
+
+        The brand guide is sent alongside the row JSON on every batch, so the LLM evaluates each
+        description with the same reference document in view.
         """
 
         assertion_type = _get_fn_name()
@@ -11241,6 +11306,13 @@ def prompt(
         if not isinstance(max_concurrent, int) or max_concurrent < 1:
             raise ValueError("max_concurrent must be a positive integer")
 
+        # Coerce `attachments=` into a list of chatlas Content objects. Fails fast
+        # on unsupported extensions so users see the error at step definition time,
+        # not deep inside interrogation.
+        from pointblank._utils_ai import _prepare_attachments
+
+        prepared_attachments = _prepare_attachments(attachments)
+
         _check_pre(pre=pre)
         _check_thresholds(thresholds=thresholds)
         _check_active_input(param=active, param_name="active")
@@ -11264,6 +11336,7 @@ def prompt(
             "llm_model": model_name,
             "batch_size": batch_size,
             "max_concurrent": max_concurrent,
+            "attachments": prepared_attachments,
         }
 
         val_info = _ValidationInfo(

pointblank/validate.pyi

@@ -419,6 +419,7 @@ class Validate:
         prompt: str,
         model: str,
         columns_subset: str | list[str] | None = None,
+        attachments: list | None = None,
         batch_size: int = 1000,
         max_concurrent: int = 3,
         pre: Callable | None = None,