feat(render): compact search results and bounded page contents

README.md

@@ -70,6 +70,13 @@ under its page (`Image: <url>` in text, an inline image in markdown).
 Tavily's `include_images` is not used: it returns image URLs without the page
 each one came from.
 
+`tinysearch_bus::render::RenderOptions` tunes the renderings: `excerpt_chars`
+and `contents_chars` set the snippet and per-page caps, and `compact_sources`
+leaves out `Sources` entries that repeat a listed result (`RenderOptions::compact()`
+turns on both that and a 1,500-character page cap). `render_text_with` and
+`render_markdown_with` take the options; `render_text` and `render_markdown`
+keep the defaults, the original output.
+
 `all_tools` exposes every available provider tool, `one_provider` exposes one
 provider's tools, and `router` exposes one `search` tool that runs the chosen
 provider's first tool with Search role normalization, defaulting to the search
@@ -91,7 +98,9 @@ classify the failure, the message starts with `tinysearch.<code>: `, and
 A direct provider requires its own credential unless it is explicitly keyless.
 A backend route requires a backend credential. Search can be disabled globally.
 Provider configuration includes a route, optional base URL, credential,
-`max_results`, `timeout_secs`, and a SearXNG `default_language`.
+`max_results`, `timeout_secs`, a SearXNG `default_language`, and
+`contents_max_chars`, which caps each page an Exa contents call returns
+(unset asks for whole pages; at most 1,500 is kept either way).
 Debug output redacts credentials.
 
 ```sh

crates/tinysearch-bus/src/render/mod.rs

@@ -10,15 +10,16 @@
 //! URL (`Image: <url>` in text, an inline image in markdown), so image search
 //! output keeps both the image and the page it came from.
 
+use std::collections::HashSet;
 use std::fmt::Write as _;
 
 use serde_json::Value;
 
 use crate::labels::provider_label;
-use crate::{ExecuteToolResponse, IMAGE_ASSETS, Role, SearchStatus};
+use crate::{Citation, ExecuteToolResponse, IMAGE_ASSETS, Role, SearchStatus};
 
-/// Snippets in rendered output are capped at this many characters.
-pub const TEXT_EXCERPT_CHARS: usize = 500;
+mod options;
+pub use options::{COMPACT_CONTENTS_CHARS, RenderOptions, TEXT_EXCERPT_CHARS};
 
 /// What the call was about: the query, the URLs of a contents call, or the
 /// objective, taken from the tool arguments.
@@ -116,9 +117,50 @@ fn result_images(response: &ExecuteToolResponse) -> Vec<Option<&str>> {
         .collect()
 }
 
+/// The citations to list under `Sources`, with their 1-based numbers.
+///
+/// All of them unless `options.compact_sources` is set; then only those that
+/// are not among the first `max_results` results, at most `max_results`.
+fn sources<'a>(
+    response: &'a ExecuteToolResponse,
+    max_results: usize,
+    options: &RenderOptions,
+) -> Vec<(usize, &'a Citation)> {
+    let numbered = response
+        .citations
+        .iter()
+        .enumerate()
+        .map(|(i, c)| (i + 1, c));
+    if !options.compact_sources {
+        return numbered.collect();
+    }
+    let shown: HashSet<&str> = response
+        .results
+        .iter()
+        .take(max_results)
+        .map(|result| result.url.trim())
+        .collect();
+    numbered
+        .filter(|(_, citation)| !shown.contains(citation.url.trim()))
+        .take(max_results)
+        .collect()
+}
+
 /// Plain-text rendering, showing at most `max_results` results.
 #[must_use]
 pub fn render_text(response: &ExecuteToolResponse, subject: &str, max_results: usize) -> String {
+    render_text_with(response, subject, max_results, &RenderOptions::default())
+}
+
+/// [`render_text`] with explicit [`RenderOptions`].
+#[must_use]
+pub fn render_text_with(
+    response: &ExecuteToolResponse,
+    subject: &str,
+    max_results: usize,
+    options: &RenderOptions,
+) -> String {
+    let excerpt_chars = options.excerpt_for(response.role);
     let mut lines = vec![heading(response, subject)];
     if let Some(answer) = non_empty(response.answer.as_deref()) {
         lines.push(String::new());
@@ -140,16 +182,17 @@ pub fn render_text(response: &ExecuteToolResponse, subject: &str, max_results: u
             lines.push(format!("   Published: {date}"));
         }
         if let Some(snippet) = non_empty(item.snippet.as_deref()) {
-            lines.push(format!("   {}", truncate(snippet, TEXT_EXCERPT_CHARS, "…")));
+            lines.push(format!("   {}", truncate(snippet, excerpt_chars, "…")));
         }
     }
-    if !response.citations.is_empty() {
+    let sources = sources(response, max_results, options);
+    if !sources.is_empty() {
         lines.push(String::new());
         lines.push("Sources:".into());
-        for (index, citation) in response.citations.iter().enumerate() {
+        for (number, citation) in sources {
             match non_empty(citation.title.as_deref()) {
-                Some(title) => lines.push(format!("[{}] {} — {}", index + 1, title, citation.url)),
-                None => lines.push(format!("[{}] {}", index + 1, citation.url)),
+                Some(title) => lines.push(format!("[{number}] {title} — {}", citation.url)),
+                None => lines.push(format!("[{number}] {}", citation.url)),
             }
         }
     }
@@ -163,6 +206,18 @@ pub fn render_markdown(
     subject: &str,
     max_results: usize,
 ) -> String {
+    render_markdown_with(response, subject, max_results, &RenderOptions::default())
+}
+
+/// [`render_markdown`] with explicit [`RenderOptions`].
+#[must_use]
+pub fn render_markdown_with(
+    response: &ExecuteToolResponse,
+    subject: &str,
+    max_results: usize,
+    options: &RenderOptions,
+) -> String {
+    let excerpt_chars = options.excerpt_for(response.role);
     let mut out = format!("# {}\n", heading(response, subject));
     if let Some(answer) = non_empty(response.answer.as_deref()) {
         let _ = write!(out, "\n{answer}\n");
@@ -179,15 +234,16 @@ pub fn render_markdown(
             let _ = write!(out, "_Published: {date}_\n\n");
         }
         if let Some(snippet) = non_empty(item.snippet.as_deref()) {
-            let _ = writeln!(out, "> {}", truncate(snippet, TEXT_EXCERPT_CHARS, "…"));
+            let _ = writeln!(out, "> {}", truncate(snippet, excerpt_chars, "…"));
         }
         if let Some(image) = images[index] {
             let _ = write!(out, "\n![{title}]({image})\n");
         }
     }
-    if !response.citations.is_empty() {
+    let sources = sources(response, max_results, options);
+    if !sources.is_empty() {
         out.push_str("\n### Sources\n");
-        for citation in &response.citations {
+        for (_, citation) in sources {
             let title = non_empty(citation.title.as_deref()).unwrap_or(citation.url.as_str());
             let _ = writeln!(out, "- [{title}]({})", citation.url);
         }

crates/tinysearch-bus/src/render/mod_tests.rs

@@ -166,3 +166,143 @@ fn an_image_search_without_images_says_so() {
         "No images found for: red panda (via Brave)"
     );
 }
+
+fn listed(count: usize) -> ExecuteToolResponse {
+    let mut many = response(Some(Role::Search));
+    many.results = (0..count)
+        .map(|i| SearchResult {
+            title: format!("r{i}"),
+            url: format!("https://e.example/{i}"),
+            snippet: None,
+            published: None,
+        })
+        .collect();
+    many.citations = many
+        .results
+        .iter()
+        .map(|r| Citation {
+            url: r.url.clone(),
+            title: Some(r.title.clone()),
+        })
+        .collect();
+    many
+}
+
+#[test]
+fn default_options_keep_the_sources_block() {
+    let text = render_text(&listed(5), "q", 5);
+    assert!(text.contains("Sources:"));
+    assert!(render_markdown(&listed(5), "q", 5).contains("### Sources"));
+}
+
+#[test]
+fn compact_drops_sources_that_repeat_every_listed_result() {
+    let options = RenderOptions::compact();
+    let text = render_text_with(&listed(5), "q", 5, &options);
+    assert!(!text.contains("Sources:"));
+    assert_eq!(text.matches("https://e.example/0").count(), 1);
+    let markdown = render_markdown_with(&listed(5), "q", 5, &options);
+    assert!(!markdown.contains("### Sources"));
+    assert_eq!(markdown.matches("https://e.example/0").count(), 1);
+}
+
+#[test]
+fn compact_still_shows_a_citation_that_is_not_a_result() {
+    let mut extra = listed(2);
+    extra.citations.push(Citation {
+        url: "https://other.example/page".into(),
+        title: Some("Other".into()),
+    });
+    let options = RenderOptions::compact();
+    let text = render_text_with(&extra, "q", 5, &options);
+    assert!(text.contains("Sources:"));
+    assert!(text.contains("[3] Other — https://other.example/page"));
+    assert_eq!(text.matches("https://e.example/0").count(), 1);
+    let markdown = render_markdown_with(&extra, "q", 5, &options);
+    assert!(markdown.contains("- [Other](https://other.example/page)"));
+    assert_eq!(markdown.matches("https://e.example/0").count(), 1);
+}
+
+#[test]
+fn compact_treats_a_result_hidden_by_max_results_as_not_listed() {
+    let text = render_text_with(&listed(4), "q", 2, &RenderOptions::compact());
+    assert!(!text.contains("[1]"));
+    assert!(text.contains("[3] r2 — https://e.example/2"));
+    assert!(text.contains("[4] r3 — https://e.example/3"));
+}
+
+#[test]
+fn compact_caps_the_sources_at_max_results() {
+    let mut answer = answer_fixture();
+    answer.citations = (0..6)
+        .map(|i| Citation {
+            url: format!("https://c.example/{i}"),
+            title: None,
+        })
+        .collect();
+    let options = RenderOptions::compact();
+    let text = render_text_with(&answer, "q", 3, &options);
+    assert!(text.contains("[3] https://c.example/2"));
+    assert!(!text.contains("https://c.example/3"));
+    let markdown = render_markdown_with(&answer, "q", 3, &options);
+    assert!(markdown.contains("https://c.example/2"));
+    assert!(!markdown.contains("https://c.example/3"));
+}
+
+#[test]
+fn excerpt_length_is_configurable() {
+    let mut long = response(Some(Role::Search));
+    long.results[0].snippet = Some("x".repeat(600));
+    let options = RenderOptions {
+        excerpt_chars: 100,
+        ..RenderOptions::default()
+    };
+    let text = render_text_with(&long, "q", 5, &options);
+    assert!(text.contains(&format!("{}…", "x".repeat(100))));
+    assert!(!text.contains(&"x".repeat(101)));
+    let markdown = render_markdown_with(&long, "q", 5, &options);
+    assert!(markdown.contains(&format!("{}…", "x".repeat(100))));
+    assert!(!markdown.contains(&"x".repeat(101)));
+}
+
+#[test]
+fn contents_pages_are_bounded_by_the_contents_cap() {
+    let mut page = response(Some(Role::Contents));
+    page.results[0].snippet = Some("y".repeat(4000));
+    let options = RenderOptions {
+        contents_chars: 1500,
+        ..RenderOptions::default()
+    };
+    let text = render_text_with(&page, "https://tokio.rs/select", 5, &options);
+    assert!(text.contains(&format!("{}…", "y".repeat(1500))));
+    assert!(!text.contains(&"y".repeat(1501)));
+    // The search excerpt cap does not apply to contents.
+    let narrow = RenderOptions {
+        excerpt_chars: 50,
+        contents_chars: 200,
+        ..RenderOptions::default()
+    };
+    let text = render_text_with(&page, "u", 5, &narrow);
+    assert!(text.contains(&"y".repeat(200)));
+    assert!(!text.contains(&"y".repeat(201)));
+}
+
+#[test]
+fn the_default_contents_cap_matches_todays_output() {
+    let mut page = response(Some(Role::Contents));
+    page.results[0].snippet = Some("y".repeat(900));
+    let text = render_text(&page, "u", 5);
+    assert!(text.contains(&format!("{}…", "y".repeat(500))));
+    assert!(!text.contains(&"y".repeat(501)));
+}
+
+#[test]
+fn render_options_round_trip_and_default_missing_fields() -> Result<(), serde_json::Error> {
+    let parsed: RenderOptions = serde_json::from_str(r#"{"compact_sources":true}"#)?;
+    assert!(parsed.compact_sources);
+    assert_eq!(parsed.excerpt_chars, TEXT_EXCERPT_CHARS);
+    assert_eq!(parsed.contents_chars, TEXT_EXCERPT_CHARS);
+    assert_eq!(RenderOptions::compact().contents_chars, 1500);
+    assert_eq!(RenderOptions::default(), serde_json::from_str("{}")?);
+    Ok(())
+}

crates/tinysearch-bus/src/render/options.rs

@@ -0,0 +1,56 @@
+//! Knobs for [`render_text_with`](super::render_text_with) and
+//! [`render_markdown_with`](super::render_markdown_with).
+
+use serde::{Deserialize, Serialize};
+
+/// Snippets in rendered output are capped at this many characters.
+pub const TEXT_EXCERPT_CHARS: usize = 500;
+
+/// Per-page character cap [`RenderOptions::compact`] uses for page contents.
+pub const COMPACT_CONTENTS_CHARS: usize = 1_500;
+
+/// How much a rendering shows. [`Default`] is the original output: every
+/// citation listed under `Sources`, snippets cut at [`TEXT_EXCERPT_CHARS`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
+#[serde(default)]
+pub struct RenderOptions {
+    /// Cap, in characters, on each result's snippet.
+    pub excerpt_chars: usize,
+    /// Cap, in characters, on each page's text in a page-contents response.
+    pub contents_chars: usize,
+    /// Leave out citations that repeat a listed result, and show at most
+    /// `max_results` of the rest.
+    pub compact_sources: bool,
+}
+
+impl Default for RenderOptions {
+    fn default() -> Self {
+        Self {
+            excerpt_chars: TEXT_EXCERPT_CHARS,
+            contents_chars: TEXT_EXCERPT_CHARS,
+            compact_sources: false,
+        }
+    }
+}
+
+impl RenderOptions {
+    /// The compact rendering: no repeated sources and
+    /// [`COMPACT_CONTENTS_CHARS`] per page.
+    #[must_use]
+    pub const fn compact() -> Self {
+        Self {
+            excerpt_chars: TEXT_EXCERPT_CHARS,
+            contents_chars: COMPACT_CONTENTS_CHARS,
+            compact_sources: true,
+        }
+    }
+
+    /// The snippet cap for a response of `role`.
+    pub(super) const fn excerpt_for(&self, role: Option<crate::Role>) -> usize {
+        match role {
+            Some(crate::Role::Contents) => self.contents_chars,
+            _ => self.excerpt_chars,
+        }
+    }
+}

crates/tinysearch-bus/src/search/mod_tests.rs

@@ -56,8 +56,9 @@ fn search_result_publication_is_optional_on_the_wire() -> serde_json::Result<()>
 fn provider_limits_round_trip_without_exposing_credential_in_debug() -> serde_json::Result<()> {
     let config: ProviderConfig = serde_json::from_value(serde_json::json!({
         "credential":"private-key", "max_results":12, "timeout_secs":8,
-        "default_language":"fr"
+        "default_language":"fr", "contents_max_chars":1500
     }))?;
+    assert_eq!(config.contents_max_chars, Some(1500));
     assert_eq!(config.max_results, Some(12));
     assert_eq!(config.timeout_secs, Some(8));
     assert_eq!(config.default_language.as_deref(), Some("fr"));
@@ -71,6 +72,7 @@ fn provider_limits_round_trip_without_exposing_credential_in_debug() -> serde_js
     assert_eq!(legacy.max_results, None);
     assert_eq!(legacy.timeout_secs, None);
     assert_eq!(legacy.default_language, None);
+    assert_eq!(legacy.contents_max_chars, None);
     Ok(())
 }