From 7f26140485e9ed445bf0212ad9d7ec438a0bcdf4 Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Sat, 17 Jan 2026 16:46:55 -0500 Subject: [PATCH 1/4] Instrumentation as comments in production --- jinjatest/__init__.py | 15 ++ jinjatest/markers.py | 179 +++++++++++++++++++ jinjatest/spec.py | 53 +++++- tests/test_markers.py | 406 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 644 insertions(+), 9 deletions(-) create mode 100644 jinjatest/markers.py create mode 100644 tests/test_markers.py diff --git a/jinjatest/__init__.py b/jinjatest/__init__.py index b95cb02..170a43e 100644 --- a/jinjatest/__init__.py +++ b/jinjatest/__init__.py @@ -35,6 +35,14 @@ def test_welcome_pro_user(): create_instrumentation, instrument, ) +from jinjatest.markers import ( + MarkerTransform, + TemplateMarkers, + discover_markers, + has_markers, + load_template_with_markers, + transform_markers, +) from jinjatest.parsers import ( FencedBlock, JSONParseError, @@ -94,6 +102,13 @@ def test_welcome_pro_user(): "ProductionInstrumentation", "TraceRecorder", "AnchorIndex", + # Markers (comment-based) + "transform_markers", + "has_markers", + "discover_markers", + "load_template_with_markers", + "MarkerTransform", + "TemplateMarkers", # Utilities "normalize_text", ] diff --git a/jinjatest/markers.py b/jinjatest/markers.py new file mode 100644 index 0000000..2d53439 --- /dev/null +++ b/jinjatest/markers.py @@ -0,0 +1,179 @@ +""" +Comment-based markers for Jinja templates. + +This module provides functionality to transform Jinja comment markers into +instrumentation function calls, enabling jinjatest to be a dev-only dependency. + +Marker syntax: + {#jt:anchor:#} - Section anchor marker + {#jt:trace:#} - Trace event marker + +Where must be a valid Python identifier: [a-zA-Z_][a-zA-Z0-9_]* +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple + +if TYPE_CHECKING: + from jinja2 import Environment, Template + + from jinjatest.instrumentation import TestInstrumentation + +# Regex patterns for comment markers +# Name must be a valid Python identifier +ANCHOR_PATTERN = re.compile(r"\{#jt:anchor:([a-zA-Z_][a-zA-Z0-9_]*)#\}") +TRACE_PATTERN = re.compile(r"\{#jt:trace:([a-zA-Z_][a-zA-Z0-9_]*)#\}") + + +class MarkerTransform(NamedTuple): + """Result of transforming a template source.""" + + source: str + anchor_names: list[str] + trace_names: list[str] + + +@dataclass +class TemplateMarkers: + """Information about markers in a template.""" + + anchors: list[str] + traces: list[str] + + @property + def has_markers(self) -> bool: + """Check if any markers were found.""" + return bool(self.anchors or self.traces) + + +def transform_markers(source: str) -> MarkerTransform: + """ + Transform comment-based markers into jt function calls. + + Args: + source: Raw template source with comment markers + + Returns: + MarkerTransform with transformed source and discovered marker names + + Example: + >>> result = transform_markers("{#jt:anchor:intro#}\\nHello") + >>> '{{ jt.anchor("intro") }}' in result.source + True + >>> result.anchor_names + ['intro'] + """ + anchor_names: list[str] = [] + trace_names: list[str] = [] + + def replace_anchor(match: re.Match[str]) -> str: + name = match.group(1) + anchor_names.append(name) + return '{{ jt.anchor("' + name + '") }}' + + def replace_trace(match: re.Match[str]) -> str: + name = match.group(1) + trace_names.append(name) + return '{{ jt.trace("' + name + '") }}' + + transformed = ANCHOR_PATTERN.sub(replace_anchor, source) + transformed = TRACE_PATTERN.sub(replace_trace, transformed) + + return MarkerTransform( + source=transformed, + anchor_names=anchor_names, + trace_names=trace_names, + ) + + +def has_markers(source: str) -> bool: + """ + Check if source contains any jt comment markers. + + Args: + source: Template source to check + + Returns: + True if any markers are found, False otherwise + + Example: + >>> has_markers("{#jt:anchor:x#}") + True + >>> has_markers("regular {# comment #}") + False + """ + return bool(ANCHOR_PATTERN.search(source) or TRACE_PATTERN.search(source)) + + +def discover_markers(source: str) -> TemplateMarkers: + """ + Discover all jt markers in a template without transforming. + + Useful for: + - Validating marker names + - Generating documentation + - CI checks for marker consistency + + Args: + source: Template source to scan + + Returns: + TemplateMarkers with lists of anchor and trace names + + Example: + >>> markers = discover_markers("{#jt:anchor:system#}\\n{#jt:trace:debug#}") + >>> markers.anchors + ['system'] + >>> markers.traces + ['debug'] + """ + anchors = ANCHOR_PATTERN.findall(source) + traces = TRACE_PATTERN.findall(source) + return TemplateMarkers(anchors=anchors, traces=traces) + + +def load_template_with_markers( + env: Environment, + template_path: str, + instrumentation: TestInstrumentation | None = None, +) -> Template: + """ + Load a template from an environment, transforming comment markers. + + This allows using comment markers with any Jinja environment. + + Args: + env: The Jinja environment (should already have jt global if instrumentation needed) + template_path: Path to template (relative to env's loader) + instrumentation: Optional instrumentation instance for trace capture + + Returns: + Compiled Template with markers transformed + + Raises: + ValueError: If the environment has no loader configured + + Example: + from jinja2 import Environment, FileSystemLoader + from jinjatest import instrument + from jinjatest.markers import load_template_with_markers + + env = Environment(loader=FileSystemLoader("templates/")) + inst = instrument(env) + template = load_template_with_markers(env, "my_prompt.j2", inst) + result = template.render({"name": "World"}) + """ + if env.loader is None: + raise ValueError("Environment must have a loader configured") + + # Get the raw source from the environment's loader + source, _, _ = env.loader.get_source(env, template_path) + + # Transform markers + transform_result = transform_markers(source) + + # Compile from transformed source + return env.from_string(transform_result.source) diff --git a/jinjatest/spec.py b/jinjatest/spec.py index a3a37d9..a1f6f33 100644 --- a/jinjatest/spec.py +++ b/jinjatest/spec.py @@ -30,6 +30,7 @@ create_instrumentation, instrument, ) +from jinjatest.markers import transform_markers from jinjatest.rendered import RenderedPrompt # Type alias for instrumentation @@ -181,6 +182,7 @@ def __init__( env: Environment, context_model: type[TContext] | None = None, instrumentation: Instrumentation | None = None, + source: str | None = None, ) -> None: """Initialize a TemplateSpec. @@ -189,6 +191,7 @@ def __init__( env: The Jinja Environment. context_model: Optional Pydantic model for context validation. instrumentation: Optional instrumentation for anchors/traces. + source: Optional original template source (used for AST analysis). """ self._template = template self._env = env @@ -196,6 +199,7 @@ def __init__( self._instrumentation: Instrumentation = ( instrumentation or create_instrumentation(test_mode=True) ) + self._source = source @classmethod def from_string( @@ -205,6 +209,7 @@ def from_string( context_model: type[TContext] | None = None, env: Environment | None = None, test_mode: bool = True, + use_comment_markers: bool = True, **env_kwargs: Any, ) -> TemplateSpec[TContext]: """Create a TemplateSpec from a template string. @@ -214,11 +219,18 @@ def from_string( context_model: Optional Pydantic model for context validation. env: Optional pre-configured Environment. test_mode: If True, enable instrumentation. + use_comment_markers: If True, transform {#jt:...#} comments to function calls. + Only applies when test_mode is True. Default True. **env_kwargs: Arguments passed to create_environment if env is None. Returns: A configured TemplateSpec. """ + # Transform comment markers if enabled and in test mode + if use_comment_markers and test_mode: + transform_result = transform_markers(source) + source = transform_result.source + if env is None: env = create_environment(**env_kwargs) @@ -240,6 +252,7 @@ def from_file( context_model: type[TContext] | None = None, env: Environment | None = None, test_mode: bool = True, + use_comment_markers: bool = True, template_dir: str | Path | None = None, **env_kwargs: Any, ) -> TemplateSpec[TContext]: @@ -250,6 +263,8 @@ def from_file( context_model: Optional Pydantic model for context validation. env: Optional pre-configured Environment. test_mode: If True, enable instrumentation. + use_comment_markers: If True, transform {#jt:...#} comments to function calls. + Only applies when test_mode is True. Default True. template_dir: Base directory for template loading. If None, uses parent of path. **env_kwargs: Arguments passed to create_environment if env is None. @@ -273,15 +288,29 @@ def from_file( instrumentation = instrument(env, test_mode=test_mode) - # Get template name relative to loader - template_name = path.name - template = env.get_template(template_name) + # If using comment markers and in test mode, read and transform the source + original_source: str | None = None + if use_comment_markers and test_mode: + # Read the raw source from file + full_path = path if path.is_absolute() else Path(template_dir or ".") / path + original_source = full_path.read_text() + + # Transform markers + transform_result = transform_markers(original_source) + + # Compile from transformed source + template = env.from_string(transform_result.source) + else: + # Get template name relative to loader + template_name = path.name + template = env.get_template(template_name) return cls( template, env=env, context_model=context_model, instrumentation=instrumentation if test_mode else None, + source=original_source, ) @property @@ -434,14 +463,20 @@ def get_undeclared_variables(self) -> set[str]: Returns: Set of variable names used in the template. """ - template_name = self._template.name - if not self._env.loader or not template_name: - # For templates from string, we need to access the source differently - # This is a limitation - we can't easily get source from compiled template - return set() + source: str | None = None + + # First, try to use stored source (from comment marker transformation) + if self._source: + source = self._source + else: + # Fall back to loading from template loader + template_name = self._template.name + if self._env.loader and template_name: + source = self._env.loader.get_source(self._env, template_name)[0] - source = self._env.loader.get_source(self._env, template_name)[0] if not source: + # For templates from string without stored source, + # we can't easily get source from compiled template return set() ast = self._env.parse(source) diff --git a/tests/test_markers.py b/tests/test_markers.py new file mode 100644 index 0000000..dc0b3b6 --- /dev/null +++ b/tests/test_markers.py @@ -0,0 +1,406 @@ +"""Tests for comment-based markers functionality.""" + +import tempfile +from pathlib import Path + +import pytest +from jinja2 import Environment, FileSystemLoader + +from jinjatest import ( + TemplateSpec, + discover_markers, + has_markers, + instrument, + load_template_with_markers, + transform_markers, +) + + +class TestMarkerTransformation: + """Tests for the transform_markers function.""" + + def test_transform_anchor(self) -> None: + """Test transforming a single anchor marker.""" + source = "{#jt:anchor:intro#}\nHello" + result = transform_markers(source) + assert '{{ jt.anchor("intro") }}' in result.source + assert result.anchor_names == ["intro"] + assert result.trace_names == [] + + def test_transform_trace(self) -> None: + """Test transforming a single trace marker.""" + source = "{% if x %}{#jt:trace:x_true#}{% endif %}" + result = transform_markers(source) + assert '{{ jt.trace("x_true") }}' in result.source + assert result.trace_names == ["x_true"] + assert result.anchor_names == [] + + def test_transform_multiple(self) -> None: + """Test transforming multiple markers.""" + source = "{#jt:anchor:a#}\n{#jt:anchor:b#}\n{#jt:trace:t#}" + result = transform_markers(source) + assert result.anchor_names == ["a", "b"] + assert result.trace_names == ["t"] + assert '{{ jt.anchor("a") }}' in result.source + assert '{{ jt.anchor("b") }}' in result.source + assert '{{ jt.trace("t") }}' in result.source + + def test_no_markers(self) -> None: + """Test source without markers passes through unchanged.""" + source = "Hello {{ name }}" + result = transform_markers(source) + assert result.source == source + assert result.anchor_names == [] + assert result.trace_names == [] + + def test_invalid_marker_names_ignored(self) -> None: + """Test that invalid identifiers are not transformed.""" + # These should NOT be transformed (invalid identifier - starts with number) + source = "{#jt:anchor:123invalid#}" + result = transform_markers(source) + assert result.source == source # Unchanged + assert result.anchor_names == [] + + def test_invalid_marker_with_hyphen_ignored(self) -> None: + """Test that identifiers with hyphens are not transformed.""" + source = "{#jt:anchor:my-anchor#}" + result = transform_markers(source) + assert result.source == source # Unchanged + + def test_valid_identifier_with_underscore(self) -> None: + """Test that underscores in identifiers work.""" + source = "{#jt:anchor:my_anchor#}" + result = transform_markers(source) + assert '{{ jt.anchor("my_anchor") }}' in result.source + assert result.anchor_names == ["my_anchor"] + + def test_valid_identifier_with_numbers(self) -> None: + """Test that numbers after first character work.""" + source = "{#jt:anchor:section2#}" + result = transform_markers(source) + assert '{{ jt.anchor("section2") }}' in result.source + + def test_marker_inline_with_content(self) -> None: + """Test markers can be inline with other content.""" + source = "Start {#jt:anchor:mid#} End" + result = transform_markers(source) + assert result.source == 'Start {{ jt.anchor("mid") }} End' + + def test_preserve_surrounding_jinja(self) -> None: + """Test that surrounding Jinja syntax is preserved.""" + source = """{% if show %} +{#jt:anchor:content#} +{{ value }} +{% endif %}""" + result = transform_markers(source) + assert "{% if show %}" in result.source + assert "{% endif %}" in result.source + assert "{{ value }}" in result.source + assert '{{ jt.anchor("content") }}' in result.source + + +class TestHasMarkers: + """Tests for the has_markers function.""" + + def test_has_anchor_marker(self) -> None: + """Test detecting anchor markers.""" + assert has_markers("{#jt:anchor:x#}") + + def test_has_trace_marker(self) -> None: + """Test detecting trace markers.""" + assert has_markers("{#jt:trace:y#}") + + def test_regular_comment_no_marker(self) -> None: + """Test that regular Jinja comments are not detected.""" + assert not has_markers("regular {# comment #}") + + def test_function_call_no_marker(self) -> None: + """Test that function call syntax is not detected as marker.""" + assert not has_markers("{{ jt.anchor('x') }}") + + def test_empty_string(self) -> None: + """Test empty string has no markers.""" + assert not has_markers("") + + def test_mixed_content(self) -> None: + """Test mixed content with marker.""" + assert has_markers("Hello {#jt:anchor:test#} World") + + +class TestDiscoverMarkers: + """Tests for the discover_markers function.""" + + def test_discover_all(self) -> None: + """Test discovering all markers in a template.""" + source = """ + {#jt:anchor:system#} + System content + {#jt:anchor:user#} + User content + {% if x %}{#jt:trace:x_branch#}{% endif %} + """ + markers = discover_markers(source) + assert markers.anchors == ["system", "user"] + assert markers.traces == ["x_branch"] + assert markers.has_markers + + def test_discover_empty(self) -> None: + """Test discovering markers in template without any.""" + markers = discover_markers("Hello {{ name }}") + assert markers.anchors == [] + assert markers.traces == [] + assert not markers.has_markers + + def test_discover_only_anchors(self) -> None: + """Test discovering only anchor markers.""" + markers = discover_markers("{#jt:anchor:a#}{#jt:anchor:b#}") + assert markers.anchors == ["a", "b"] + assert markers.traces == [] + assert markers.has_markers + + def test_discover_only_traces(self) -> None: + """Test discovering only trace markers.""" + markers = discover_markers("{#jt:trace:t1#}{#jt:trace:t2#}") + assert markers.anchors == [] + assert markers.traces == ["t1", "t2"] + assert markers.has_markers + + +class TestTemplateSpecWithMarkers: + """Tests for TemplateSpec integration with comment markers.""" + + def test_from_string_transforms_markers(self) -> None: + """Test that from_string transforms comment markers.""" + spec = TemplateSpec.from_string("{#jt:anchor:test#}\nHello {{ name }}") + rendered = spec.render({"name": "World"}) + assert rendered.has_section("test") + assert rendered.section("test").contains("Hello World") + + def test_anchor_sections_work(self) -> None: + """Test that transformed anchor markers create working sections.""" + spec = TemplateSpec.from_string( + """{#jt:anchor:system#} +You are a helpful assistant. + +{#jt:anchor:user#} +User: {{ user_input }}""" + ) + rendered = spec.render({"user_input": "Hello!"}) + + assert rendered.has_section("system") + assert rendered.has_section("user") + assert "helpful assistant" in rendered.section("system").text + assert "User: Hello!" in rendered.section("user").text + + def test_trace_in_conditional(self) -> None: + """Test that transformed trace markers work in conditionals.""" + spec = TemplateSpec.from_string( + """{% if show_extra %} +{#jt:trace:extra_shown#} +Extra content +{% endif %}""" + ) + + rendered = spec.render({"show_extra": True}) + assert rendered.has_trace("extra_shown") + + rendered = spec.render({"show_extra": False}) + assert not rendered.has_trace("extra_shown") + + def test_disable_marker_transformation(self) -> None: + """Test disabling marker transformation.""" + spec = TemplateSpec.from_string( + "{#jt:anchor:test#}\nHello", + use_comment_markers=False, + ) + rendered = spec.render({}) + # Marker was not transformed, so no section exists + assert not rendered.has_section("test") + # The comment is stripped by Jinja since it wasn't transformed + assert "Hello" in rendered.text + + def test_markers_not_in_output(self) -> None: + """Test that marker comments don't appear in production output.""" + # In test mode with markers, the markers are transformed to function calls + # The anchor markers emit sentinel strings that are indexed but not visible + spec = TemplateSpec.from_string("{#jt:anchor:test#}Hello World") + rendered = spec.render({}) + # The clean_text should not contain the comment syntax + assert "{#jt:" not in rendered.clean_text + assert "Hello World" in rendered.clean_text + + def test_mixed_markers_and_function_calls(self) -> None: + """Test mixing comment markers with function call syntax.""" + spec = TemplateSpec.from_string( + """{#jt:anchor:comment_style#} +Comment-based section + +{{ jt.anchor("function_style") }} +Function-based section""" + ) + rendered = spec.render({}) + assert rendered.has_section("comment_style") + assert rendered.has_section("function_style") + + def test_multiple_anchors_ordering(self) -> None: + """Test that multiple anchors maintain correct ordering.""" + spec = TemplateSpec.from_string( + """{#jt:anchor:first#} +First section +{#jt:anchor:second#} +Second section +{#jt:anchor:third#} +Third section""" + ) + rendered = spec.render({}) + + sections = rendered.sections() + assert list(sections.keys()) == ["first", "second", "third"] + + def test_trace_count(self) -> None: + """Test counting multiple trace events.""" + spec = TemplateSpec.from_string( + """{% for i in items %} +{#jt:trace:item_processed#} +Item: {{ i }} +{% endfor %}""" + ) + rendered = spec.render({"items": [1, 2, 3]}) + assert rendered.trace_count("item_processed") == 3 + + +class TestTemplateSpecFromFile: + """Tests for TemplateSpec.from_file with markers.""" + + def test_from_file_transforms_markers(self) -> None: + """Test that from_file transforms comment markers.""" + with tempfile.TemporaryDirectory() as tmpdir: + template_path = Path(tmpdir) / "test.j2" + template_path.write_text( + """{#jt:anchor:greeting#} +Hello, {{ name }}!""" + ) + + spec = TemplateSpec.from_file(template_path) + rendered = spec.render({"name": "World"}) + + assert rendered.has_section("greeting") + assert "Hello, World!" in rendered.section("greeting").text + + def test_from_file_disable_markers(self) -> None: + """Test disabling markers when loading from file.""" + with tempfile.TemporaryDirectory() as tmpdir: + template_path = Path(tmpdir) / "test.j2" + template_path.write_text("{#jt:anchor:test#}Content") + + spec = TemplateSpec.from_file(template_path, use_comment_markers=False) + rendered = spec.render({}) + + assert not rendered.has_section("test") + + +class TestLoadTemplateWithMarkers: + """Tests for the load_template_with_markers utility function.""" + + def test_load_and_transform(self) -> None: + """Test loading a template with marker transformation.""" + with tempfile.TemporaryDirectory() as tmpdir: + template_path = Path(tmpdir) / "prompt.j2" + template_path.write_text( + """{#jt:anchor:system#} +System prompt + +{#jt:anchor:user#} +{{ user_input }}""" + ) + + env = Environment(loader=FileSystemLoader(tmpdir)) + inst = instrument(env) + + template = load_template_with_markers(env, "prompt.j2", inst) + result = template.render({"user_input": "Hello"}) + + # The anchors should have been transformed and rendered + assert "System prompt" in result + assert "Hello" in result + + def test_load_without_loader_raises(self) -> None: + """Test that loading without a loader raises an error.""" + env = Environment() # No loader + + with pytest.raises(ValueError, match="must have a loader"): + load_template_with_markers(env, "template.j2") + + +class TestRealWorldScenarios: + """Integration tests for real-world usage patterns.""" + + def test_llm_prompt_template(self) -> None: + """Test a realistic LLM prompt template.""" + template = """{#jt:anchor:system#} +You are a helpful AI assistant. Be concise and accurate. + +{#jt:anchor:context#} +{% if context %} +{#jt:trace:context_provided#} +Context: {{ context }} +{% endif %} + +{#jt:anchor:user#} +User: {{ user_message }} + +{% if tools %} +{#jt:trace:tools_available#} +Available tools: {{ tools | join(", ") }} +{% endif %}""" + + spec = TemplateSpec.from_string(template) + + # Test with context and tools + rendered = spec.render( + { + "context": "User prefers short answers", + "user_message": "What is Python?", + "tools": ["search", "calculate"], + } + ) + + assert rendered.has_section("system") + assert rendered.has_section("context") + assert rendered.has_section("user") + + assert rendered.has_trace("context_provided") + assert rendered.has_trace("tools_available") + + assert "helpful AI assistant" in rendered.section("system").text + assert "User prefers short answers" in rendered.section("context").text + assert "What is Python?" in rendered.section("user").text + + # Test without context + rendered = spec.render( + { + "context": None, + "user_message": "Hello", + "tools": None, + } + ) + + assert not rendered.has_trace("context_provided") + assert not rendered.has_trace("tools_available") + + def test_production_vs_test_mode(self) -> None: + """Test behavior difference between production and test mode.""" + template = "{#jt:anchor:content#}Hello {{ name }}" + + # Test mode (default) - markers are transformed + test_spec = TemplateSpec.from_string(template, test_mode=True) + test_rendered = test_spec.render({"name": "World"}) + assert test_rendered.has_section("content") + + # Production mode - markers not transformed, but comment stripped by Jinja + prod_spec = TemplateSpec.from_string(template, test_mode=False) + prod_rendered = prod_spec.render({"name": "World"}) + # In production mode, no instrumentation so no sections + assert not prod_rendered.has_section("content") + # The content should still be there (comment stripped by Jinja) + assert "Hello World" in prod_rendered.text From 51fb239f9e1dd1483801f46ef6c0909c62dcfb87 Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Sat, 17 Jan 2026 17:12:41 -0500 Subject: [PATCH 2/4] Fixed TeplateSpec from file --- jinjatest/spec.py | 47 ++++++++++--- tests/test_basic.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 9 deletions(-) diff --git a/jinjatest/spec.py b/jinjatest/spec.py index a1f6f33..7553756 100644 --- a/jinjatest/spec.py +++ b/jinjatest/spec.py @@ -259,19 +259,26 @@ def from_file( """Create a TemplateSpec from a template file. Args: - path: Path to the template file. + path: Path to the template file. When env is provided, this should be + relative to the environment's loader. When env is None, this can be + an absolute path or relative to cwd. context_model: Optional Pydantic model for context validation. - env: Optional pre-configured Environment. + env: Optional pre-configured Environment. If provided, the path is used + as-is (relative to the env's loader). If None, a new environment + is created with the template's parent directory as the loader root. test_mode: If True, enable instrumentation. use_comment_markers: If True, transform {#jt:...#} comments to function calls. Only applies when test_mode is True. Default True. - template_dir: Base directory for template loading. If None, uses parent of path. + template_dir: Base directory for template loading. Only used when env is None. + If None, uses parent directory of path. **env_kwargs: Arguments passed to create_environment if env is None. Returns: A configured TemplateSpec. """ path = Path(path) + env_was_provided = env is not None + template_dir_was_provided = template_dir is not None if env is None: # Determine template directory @@ -285,15 +292,39 @@ def from_file( template_paths = [template_dir] + [Path(p) for p in template_paths] env = create_environment(template_paths=template_paths, **env_kwargs) + instrumentation = instrument(env, test_mode=test_mode) + else: + # For provided env, check if already instrumented + existing_jt = env.globals.get("jt") + if isinstance( + existing_jt, (TestInstrumentation, ProductionInstrumentation) + ): + instrumentation = existing_jt + else: + instrumentation = instrument(env, test_mode=test_mode) - instrumentation = instrument(env, test_mode=test_mode) + # Determine template name based on how env was obtained + # When env is provided or template_dir is explicitly set, use full path + # Otherwise use just filename (loader points to path.parent) + if env_was_provided or template_dir_was_provided: + template_name = str(path) + else: + template_name = path.name # If using comment markers and in test mode, read and transform the source original_source: str | None = None if use_comment_markers and test_mode: - # Read the raw source from file - full_path = path if path.is_absolute() else Path(template_dir or ".") / path - original_source = full_path.read_text() + if env_was_provided: + # Read from loader (for provided env) + if env.loader is None: + raise TemplateRenderError( + "Cannot use comment markers with provided env that has no loader" + ) + original_source, _, _ = env.loader.get_source(env, template_name) + else: + # Read from file system (for newly created env) + full_path = path if path.is_absolute() else Path(template_dir) / path + original_source = full_path.read_text() # Transform markers transform_result = transform_markers(original_source) @@ -301,8 +332,6 @@ def from_file( # Compile from transformed source template = env.from_string(transform_result.source) else: - # Get template name relative to loader - template_name = path.name template = env.get_template(template_name) return cls( diff --git a/tests/test_basic.py b/tests/test_basic.py index 76cd4fb..94dcb31 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1887,3 +1887,167 @@ def test_from_text_anchor_sections(self) -> None: body = index.get_section("body") assert "Body content" in body + + +class TestFromFileWithProvidedEnv: + """Tests for from_file when an environment is provided.""" + + @pytest.fixture + def template_dir(self, tmp_path: Path) -> Path: + """Create a nested template structure.""" + # Create: tmp/prompts/feature/section/template.j2 + nested = tmp_path / "prompts" / "feature" / "section" + nested.mkdir(parents=True) + + template_file = nested / "template.j2" + template_file.write_text("Hello {{ name }}!") + + return tmp_path / "prompts" + + def test_from_file_with_env_preserves_path(self, template_dir: Path) -> None: + """When env is provided, full path should be preserved.""" + from jinja2 import Environment, FileSystemLoader + from jinjatest import TemplateSpec, instrument + + env = Environment(loader=FileSystemLoader(str(template_dir))) + instrument(env, test_mode=True) + + # This should work - path relative to env's loader + spec = TemplateSpec.from_file("feature/section/template.j2", env=env) + rendered = spec.render({"name": "World"}) + + assert rendered.text == "Hello World!" + + def test_from_file_with_env_reuses_instrumentation( + self, template_dir: Path + ) -> None: + """When env is already instrumented, should reuse instrumentation.""" + from jinja2 import Environment, FileSystemLoader + from jinjatest import TemplateSpec, instrument + + env = Environment(loader=FileSystemLoader(str(template_dir))) + original_inst = instrument(env, test_mode=True) + + spec = TemplateSpec.from_file("feature/section/template.j2", env=env) + + # Should be the same instrumentation instance + assert spec._instrumentation is original_inst + + def test_from_file_without_env_uses_filename_only(self, template_dir: Path) -> None: + """When env is None, should use just filename with parent as loader.""" + template_path = template_dir / "feature" / "section" / "template.j2" + + # No env provided - should create one with loader at template's parent + spec = TemplateSpec.from_file(str(template_path)) + rendered = spec.render({"name": "World"}) + + assert rendered.text == "Hello World!" + + def test_from_file_with_env_instruments_if_needed(self, template_dir: Path) -> None: + """When env is provided but not instrumented, should add instrumentation.""" + from jinja2 import Environment, FileSystemLoader + from jinjatest import TemplateSpec + + env = Environment(loader=FileSystemLoader(str(template_dir))) + # Note: NOT calling instrument(env) here + + spec = TemplateSpec.from_file("feature/section/template.j2", env=env) + + # Should have added instrumentation + assert "jt" in env.globals + assert spec._instrumentation is not None + + +class TestFromFileWithTemplateDir: + """Tests for from_file with explicit template_dir.""" + + @pytest.fixture + def template_structure(self, tmp_path: Path) -> Path: + """Create template structure.""" + prompts = tmp_path / "prompts" + nested = prompts / "v2" / "chat" + nested.mkdir(parents=True) + + (nested / "system.j2").write_text("System: {{ mode }}") + (nested / "user.j2").write_text("User: {{ input }}") + + return prompts + + def test_from_file_with_template_dir(self, template_structure: Path) -> None: + """template_dir should set the loader root.""" + spec = TemplateSpec.from_file( + "v2/chat/system.j2", + template_dir=template_structure, + ) + rendered = spec.render({"mode": "helpful"}) + + assert rendered.text == "System: helpful" + + def test_from_file_template_dir_ignored_when_env_provided( + self, template_structure: Path + ) -> None: + """template_dir should be ignored when env is provided.""" + from jinja2 import Environment, FileSystemLoader + from jinjatest import TemplateSpec, instrument + + env = Environment(loader=FileSystemLoader(str(template_structure))) + instrument(env, test_mode=True) + + # template_dir is provided but should be ignored since env is provided + spec = TemplateSpec.from_file( + "v2/chat/system.j2", + env=env, + template_dir="/some/other/path", # Should be ignored + ) + rendered = spec.render({"mode": "helpful"}) + + assert rendered.text == "System: helpful" + + +class TestFromFileWithCommentMarkersAndProvidedEnv: + """Tests for from_file with comment markers when env is provided.""" + + @pytest.fixture + def template_with_markers(self, tmp_path: Path) -> Path: + """Create a template with comment markers in a nested structure.""" + nested = tmp_path / "prompts" / "v2" + nested.mkdir(parents=True) + + template_file = nested / "marked.j2" + template_file.write_text( + "{#jt:anchor:greeting#}Hello {{ name }}!{#jt:trace:rendered#}" + ) + + return tmp_path / "prompts" + + def test_from_file_with_env_and_markers(self, template_with_markers: Path) -> None: + """Test that comment markers work when env is provided.""" + from jinja2 import Environment, FileSystemLoader + from jinjatest import TemplateSpec, instrument + + env = Environment(loader=FileSystemLoader(str(template_with_markers))) + instrument(env, test_mode=True) + + spec = TemplateSpec.from_file("v2/marked.j2", env=env) + rendered = spec.render({"name": "World"}) + + assert "Hello World!" in rendered.text + # Check that trace was recorded + assert rendered.has_trace("rendered") + + def test_from_file_with_env_no_loader_raises(self) -> None: + """Test that error is raised when env has no loader and markers are used.""" + from jinja2 import Environment + from jinjatest import TemplateSpec, TemplateRenderError + + env = Environment() # No loader + + with pytest.raises(TemplateRenderError) as exc_info: + TemplateSpec.from_file( + "some/path/template.j2", + env=env, + use_comment_markers=True, + test_mode=True, + ) + + assert "no loader" in str(exc_info.value) From 3386c997ba2c608c837a9f171d89e4c1159d26ce Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Sat, 17 Jan 2026 17:37:44 -0500 Subject: [PATCH 3/4] Updated README.md --- README.md | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2fb875b..43176a0 100644 --- a/README.md +++ b/README.md @@ -151,23 +151,23 @@ Test specific sections without fragile delimiters: ```jinja2 {# prompts/chat.j2 #} -{{ jt.anchor("system") }} +{#jt:anchor:system#} System rules: - Be helpful - Be concise -{{ jt.anchor("user") }} +{#jt:anchor:user#} User: {{ user_name }} Request: {{ request }} -{{ jt.anchor("context") }} +{#jt:anchor:context#} {% if context_items %} Context: {% for item in context_items %} - {{ item }} {% endfor %} {% else %} -{{ jt.trace("no_context") }} +{#jt:trace:no_context#} No additional context. {% endif %} ``` @@ -232,16 +232,18 @@ def test_prompt_builder(): # From file spec = TemplateSpec.from_file( "template.j2", - context_model=MyModel, # Optional Pydantic model - template_dir="templates/", # Optional base directory - strict_undefined=True, # Default: True - test_mode=True, # Enable instrumentation + context_model=MyModel, # Optional Pydantic model + template_dir="templates/", # Optional base directory + strict_undefined=True, # Default: True + test_mode=True, # Enable instrumentation + use_comment_markers=True, # Transform {#jt:...#} comments (default: True) ) # From string spec = TemplateSpec.from_string( "Hello {{ name }}!", context_model=MyModel, + use_comment_markers=True, # Transform {#jt:...#} comments (default: True) ) # Render @@ -321,37 +323,40 @@ a.snapshot("snapshot_name", update=False) ### Instrumentation -In templates: +In templates, use comment-based markers to define sections and trace events: + ```jinja2 -{{ jt.anchor("section_name") }} {# Mark section start #} -{{ jt.trace("event_name") }} {# Record trace event #} +{#jt:anchor:section_name#} {# Mark section start #} +{#jt:trace:event_name#} {# Record trace event #} ``` +Comment markers are automatically transformed when `test_mode=True`. This allows jinjatest to be a dev-only dependency since the comments are valid Jinja syntax that render as empty strings in production. + #### Using with Any Jinja Environment You can add instrumentation to any Jinja environment using `instrument()`: ```python from jinja2 import Environment, FileSystemLoader -from jinjatest import instrument +from jinjatest import TemplateSpec, instrument # Patch any existing Jinja environment env = Environment(loader=FileSystemLoader("templates/")) -inst = instrument(env) # Adds `jt` global +instrument(env) # Adds `jt` global -# Now templates can use {{ jt.anchor("x") }} and {{ jt.trace("y") }} -template = env.get_template("my_template.j2") -result = template.render({"name": "World"}) +# Load template with comment markers transformed +spec = TemplateSpec.from_file("my_template.j2", env=env) +rendered = spec.render({"name": "World"}) # Check traces after rendering -if inst.has_trace("some_event"): +if rendered.has_trace("some_event"): print("Event was triggered") -# For production, use test_mode=False (anchors/traces become no-ops) +# For production, use test_mode=False (markers become no-ops) instrument(env, test_mode=False) ``` -This is useful when you want to add instrumentation to an existing Jinja setup without using `TemplateSpec`. +This is useful when you want to add instrumentation to an existing Jinja setup. ## Pytest Integration From 3116c8623fb7bf1be704c45b2bc229136a2b13fa Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Sat, 17 Jan 2026 17:48:42 -0500 Subject: [PATCH 4/4] Bump version to 0.1.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2ab3ca9..7002e61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "jinjatest" -version = "0.1.0" +version = "0.1.1" description = "A type-safe, structured testing library for Jinja templates" readme = "README.md" requires-python = ">=3.10"