diff --git a/README.md b/README.md index 65c029d..f9e0eb8 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,13 @@ under its page (`Image: ` in text, an inline image in markdown). Tavily's `include_images` is not used: it returns image URLs without the page each one came from. +`tinysearch_bus::render::RenderOptions` tunes the renderings: `excerpt_chars` +and `contents_chars` set the snippet and per-page caps, and `compact_sources` +leaves out `Sources` entries that repeat a listed result (`RenderOptions::compact()` +turns on both that and a 1,500-character page cap). `render_text_with` and +`render_markdown_with` take the options; `render_text` and `render_markdown` +keep the defaults, the original output. + `all_tools` exposes every available provider tool, `one_provider` exposes one provider's tools, and `router` exposes one `search` tool that runs the chosen provider's first tool with Search role normalization, defaulting to the search @@ -91,7 +98,9 @@ classify the failure, the message starts with `tinysearch.: `, and A direct provider requires its own credential unless it is explicitly keyless. A backend route requires a backend credential. Search can be disabled globally. Provider configuration includes a route, optional base URL, credential, -`max_results`, `timeout_secs`, and a SearXNG `default_language`. +`max_results`, `timeout_secs`, a SearXNG `default_language`, and +`contents_max_chars`, which caps each page an Exa contents call returns +(unset asks for whole pages; at most 1,500 is kept either way). Debug output redacts credentials. ```sh diff --git a/crates/tinysearch-bus/src/render/mod.rs b/crates/tinysearch-bus/src/render/mod.rs index c87eb42..6ffffd4 100644 --- a/crates/tinysearch-bus/src/render/mod.rs +++ b/crates/tinysearch-bus/src/render/mod.rs @@ -10,15 +10,16 @@ //! URL (`Image: ` in text, an inline image in markdown), so image search //! output keeps both the image and the page it came from. +use std::collections::HashSet; use std::fmt::Write as _; use serde_json::Value; use crate::labels::provider_label; -use crate::{ExecuteToolResponse, IMAGE_ASSETS, Role, SearchStatus}; +use crate::{Citation, ExecuteToolResponse, IMAGE_ASSETS, Role, SearchStatus}; -/// Snippets in rendered output are capped at this many characters. -pub const TEXT_EXCERPT_CHARS: usize = 500; +mod options; +pub use options::{COMPACT_CONTENTS_CHARS, RenderOptions, TEXT_EXCERPT_CHARS}; /// What the call was about: the query, the URLs of a contents call, or the /// objective, taken from the tool arguments. @@ -116,9 +117,50 @@ fn result_images(response: &ExecuteToolResponse) -> Vec> { .collect() } +/// The citations to list under `Sources`, with their 1-based numbers. +/// +/// All of them unless `options.compact_sources` is set; then only those that +/// are not among the first `max_results` results, at most `max_results`. +fn sources<'a>( + response: &'a ExecuteToolResponse, + max_results: usize, + options: &RenderOptions, +) -> Vec<(usize, &'a Citation)> { + let numbered = response + .citations + .iter() + .enumerate() + .map(|(i, c)| (i + 1, c)); + if !options.compact_sources { + return numbered.collect(); + } + let shown: HashSet<&str> = response + .results + .iter() + .take(max_results) + .map(|result| result.url.trim()) + .collect(); + numbered + .filter(|(_, citation)| !shown.contains(citation.url.trim())) + .take(max_results) + .collect() +} + /// Plain-text rendering, showing at most `max_results` results. #[must_use] pub fn render_text(response: &ExecuteToolResponse, subject: &str, max_results: usize) -> String { + render_text_with(response, subject, max_results, &RenderOptions::default()) +} + +/// [`render_text`] with explicit [`RenderOptions`]. +#[must_use] +pub fn render_text_with( + response: &ExecuteToolResponse, + subject: &str, + max_results: usize, + options: &RenderOptions, +) -> String { + let excerpt_chars = options.excerpt_for(response.role); let mut lines = vec![heading(response, subject)]; if let Some(answer) = non_empty(response.answer.as_deref()) { lines.push(String::new()); @@ -140,16 +182,17 @@ pub fn render_text(response: &ExecuteToolResponse, subject: &str, max_results: u lines.push(format!(" Published: {date}")); } if let Some(snippet) = non_empty(item.snippet.as_deref()) { - lines.push(format!(" {}", truncate(snippet, TEXT_EXCERPT_CHARS, "…"))); + lines.push(format!(" {}", truncate(snippet, excerpt_chars, "…"))); } } - if !response.citations.is_empty() { + let sources = sources(response, max_results, options); + if !sources.is_empty() { lines.push(String::new()); lines.push("Sources:".into()); - for (index, citation) in response.citations.iter().enumerate() { + for (number, citation) in sources { match non_empty(citation.title.as_deref()) { - Some(title) => lines.push(format!("[{}] {} — {}", index + 1, title, citation.url)), - None => lines.push(format!("[{}] {}", index + 1, citation.url)), + Some(title) => lines.push(format!("[{number}] {title} — {}", citation.url)), + None => lines.push(format!("[{number}] {}", citation.url)), } } } @@ -163,6 +206,18 @@ pub fn render_markdown( subject: &str, max_results: usize, ) -> String { + render_markdown_with(response, subject, max_results, &RenderOptions::default()) +} + +/// [`render_markdown`] with explicit [`RenderOptions`]. +#[must_use] +pub fn render_markdown_with( + response: &ExecuteToolResponse, + subject: &str, + max_results: usize, + options: &RenderOptions, +) -> String { + let excerpt_chars = options.excerpt_for(response.role); let mut out = format!("# {}\n", heading(response, subject)); if let Some(answer) = non_empty(response.answer.as_deref()) { let _ = write!(out, "\n{answer}\n"); @@ -179,15 +234,16 @@ pub fn render_markdown( let _ = write!(out, "_Published: {date}_\n\n"); } if let Some(snippet) = non_empty(item.snippet.as_deref()) { - let _ = writeln!(out, "> {}", truncate(snippet, TEXT_EXCERPT_CHARS, "…")); + let _ = writeln!(out, "> {}", truncate(snippet, excerpt_chars, "…")); } if let Some(image) = images[index] { let _ = write!(out, "\n![{title}]({image})\n"); } } - if !response.citations.is_empty() { + let sources = sources(response, max_results, options); + if !sources.is_empty() { out.push_str("\n### Sources\n"); - for citation in &response.citations { + for (_, citation) in sources { let title = non_empty(citation.title.as_deref()).unwrap_or(citation.url.as_str()); let _ = writeln!(out, "- [{title}]({})", citation.url); } diff --git a/crates/tinysearch-bus/src/render/mod_tests.rs b/crates/tinysearch-bus/src/render/mod_tests.rs index 5ad07bd..ceb346a 100644 --- a/crates/tinysearch-bus/src/render/mod_tests.rs +++ b/crates/tinysearch-bus/src/render/mod_tests.rs @@ -166,3 +166,143 @@ fn an_image_search_without_images_says_so() { "No images found for: red panda (via Brave)" ); } + +fn listed(count: usize) -> ExecuteToolResponse { + let mut many = response(Some(Role::Search)); + many.results = (0..count) + .map(|i| SearchResult { + title: format!("r{i}"), + url: format!("https://e.example/{i}"), + snippet: None, + published: None, + }) + .collect(); + many.citations = many + .results + .iter() + .map(|r| Citation { + url: r.url.clone(), + title: Some(r.title.clone()), + }) + .collect(); + many +} + +#[test] +fn default_options_keep_the_sources_block() { + let text = render_text(&listed(5), "q", 5); + assert!(text.contains("Sources:")); + assert!(render_markdown(&listed(5), "q", 5).contains("### Sources")); +} + +#[test] +fn compact_drops_sources_that_repeat_every_listed_result() { + let options = RenderOptions::compact(); + let text = render_text_with(&listed(5), "q", 5, &options); + assert!(!text.contains("Sources:")); + assert_eq!(text.matches("https://e.example/0").count(), 1); + let markdown = render_markdown_with(&listed(5), "q", 5, &options); + assert!(!markdown.contains("### Sources")); + assert_eq!(markdown.matches("https://e.example/0").count(), 1); +} + +#[test] +fn compact_still_shows_a_citation_that_is_not_a_result() { + let mut extra = listed(2); + extra.citations.push(Citation { + url: "https://other.example/page".into(), + title: Some("Other".into()), + }); + let options = RenderOptions::compact(); + let text = render_text_with(&extra, "q", 5, &options); + assert!(text.contains("Sources:")); + assert!(text.contains("[3] Other — https://other.example/page")); + assert_eq!(text.matches("https://e.example/0").count(), 1); + let markdown = render_markdown_with(&extra, "q", 5, &options); + assert!(markdown.contains("- [Other](https://other.example/page)")); + assert_eq!(markdown.matches("https://e.example/0").count(), 1); +} + +#[test] +fn compact_treats_a_result_hidden_by_max_results_as_not_listed() { + let text = render_text_with(&listed(4), "q", 2, &RenderOptions::compact()); + assert!(!text.contains("[1]")); + assert!(text.contains("[3] r2 — https://e.example/2")); + assert!(text.contains("[4] r3 — https://e.example/3")); +} + +#[test] +fn compact_caps_the_sources_at_max_results() { + let mut answer = answer_fixture(); + answer.citations = (0..6) + .map(|i| Citation { + url: format!("https://c.example/{i}"), + title: None, + }) + .collect(); + let options = RenderOptions::compact(); + let text = render_text_with(&answer, "q", 3, &options); + assert!(text.contains("[3] https://c.example/2")); + assert!(!text.contains("https://c.example/3")); + let markdown = render_markdown_with(&answer, "q", 3, &options); + assert!(markdown.contains("https://c.example/2")); + assert!(!markdown.contains("https://c.example/3")); +} + +#[test] +fn excerpt_length_is_configurable() { + let mut long = response(Some(Role::Search)); + long.results[0].snippet = Some("x".repeat(600)); + let options = RenderOptions { + excerpt_chars: 100, + ..RenderOptions::default() + }; + let text = render_text_with(&long, "q", 5, &options); + assert!(text.contains(&format!("{}…", "x".repeat(100)))); + assert!(!text.contains(&"x".repeat(101))); + let markdown = render_markdown_with(&long, "q", 5, &options); + assert!(markdown.contains(&format!("{}…", "x".repeat(100)))); + assert!(!markdown.contains(&"x".repeat(101))); +} + +#[test] +fn contents_pages_are_bounded_by_the_contents_cap() { + let mut page = response(Some(Role::Contents)); + page.results[0].snippet = Some("y".repeat(4000)); + let options = RenderOptions { + contents_chars: 1500, + ..RenderOptions::default() + }; + let text = render_text_with(&page, "https://tokio.rs/select", 5, &options); + assert!(text.contains(&format!("{}…", "y".repeat(1500)))); + assert!(!text.contains(&"y".repeat(1501))); + // The search excerpt cap does not apply to contents. + let narrow = RenderOptions { + excerpt_chars: 50, + contents_chars: 200, + ..RenderOptions::default() + }; + let text = render_text_with(&page, "u", 5, &narrow); + assert!(text.contains(&"y".repeat(200))); + assert!(!text.contains(&"y".repeat(201))); +} + +#[test] +fn the_default_contents_cap_matches_todays_output() { + let mut page = response(Some(Role::Contents)); + page.results[0].snippet = Some("y".repeat(900)); + let text = render_text(&page, "u", 5); + assert!(text.contains(&format!("{}…", "y".repeat(500)))); + assert!(!text.contains(&"y".repeat(501))); +} + +#[test] +fn render_options_round_trip_and_default_missing_fields() -> Result<(), serde_json::Error> { + let parsed: RenderOptions = serde_json::from_str(r#"{"compact_sources":true}"#)?; + assert!(parsed.compact_sources); + assert_eq!(parsed.excerpt_chars, TEXT_EXCERPT_CHARS); + assert_eq!(parsed.contents_chars, TEXT_EXCERPT_CHARS); + assert_eq!(RenderOptions::compact().contents_chars, 1500); + assert_eq!(RenderOptions::default(), serde_json::from_str("{}")?); + Ok(()) +} diff --git a/crates/tinysearch-bus/src/render/options.rs b/crates/tinysearch-bus/src/render/options.rs new file mode 100644 index 0000000..ad3e6b4 --- /dev/null +++ b/crates/tinysearch-bus/src/render/options.rs @@ -0,0 +1,56 @@ +//! Knobs for [`render_text_with`](super::render_text_with) and +//! [`render_markdown_with`](super::render_markdown_with). + +use serde::{Deserialize, Serialize}; + +/// Snippets in rendered output are capped at this many characters. +pub const TEXT_EXCERPT_CHARS: usize = 500; + +/// Per-page character cap [`RenderOptions::compact`] uses for page contents. +pub const COMPACT_CONTENTS_CHARS: usize = 1_500; + +/// How much a rendering shows. [`Default`] is the original output: every +/// citation listed under `Sources`, snippets cut at [`TEXT_EXCERPT_CHARS`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[serde(default)] +pub struct RenderOptions { + /// Cap, in characters, on each result's snippet. + pub excerpt_chars: usize, + /// Cap, in characters, on each page's text in a page-contents response. + pub contents_chars: usize, + /// Leave out citations that repeat a listed result, and show at most + /// `max_results` of the rest. + pub compact_sources: bool, +} + +impl Default for RenderOptions { + fn default() -> Self { + Self { + excerpt_chars: TEXT_EXCERPT_CHARS, + contents_chars: TEXT_EXCERPT_CHARS, + compact_sources: false, + } + } +} + +impl RenderOptions { + /// The compact rendering: no repeated sources and + /// [`COMPACT_CONTENTS_CHARS`] per page. + #[must_use] + pub const fn compact() -> Self { + Self { + excerpt_chars: TEXT_EXCERPT_CHARS, + contents_chars: COMPACT_CONTENTS_CHARS, + compact_sources: true, + } + } + + /// The snippet cap for a response of `role`. + pub(super) const fn excerpt_for(&self, role: Option) -> usize { + match role { + Some(crate::Role::Contents) => self.contents_chars, + _ => self.excerpt_chars, + } + } +} diff --git a/crates/tinysearch-bus/src/search/mod_tests.rs b/crates/tinysearch-bus/src/search/mod_tests.rs index 3113cb6..434b517 100644 --- a/crates/tinysearch-bus/src/search/mod_tests.rs +++ b/crates/tinysearch-bus/src/search/mod_tests.rs @@ -56,8 +56,9 @@ fn search_result_publication_is_optional_on_the_wire() -> serde_json::Result<()> fn provider_limits_round_trip_without_exposing_credential_in_debug() -> serde_json::Result<()> { let config: ProviderConfig = serde_json::from_value(serde_json::json!({ "credential":"private-key", "max_results":12, "timeout_secs":8, - "default_language":"fr" + "default_language":"fr", "contents_max_chars":1500 }))?; + assert_eq!(config.contents_max_chars, Some(1500)); assert_eq!(config.max_results, Some(12)); assert_eq!(config.timeout_secs, Some(8)); assert_eq!(config.default_language.as_deref(), Some("fr")); @@ -71,6 +72,7 @@ fn provider_limits_round_trip_without_exposing_credential_in_debug() -> serde_js assert_eq!(legacy.max_results, None); assert_eq!(legacy.timeout_secs, None); assert_eq!(legacy.default_language, None); + assert_eq!(legacy.contents_max_chars, None); Ok(()) } diff --git a/crates/tinysearch-bus/src/search/types.rs b/crates/tinysearch-bus/src/search/types.rs index 220c278..ebd1252 100644 --- a/crates/tinysearch-bus/src/search/types.rs +++ b/crates/tinysearch-bus/src/search/types.rs @@ -158,6 +158,10 @@ pub struct ProviderConfig { pub timeout_secs: Option, /// Default `SearXNG` language when the search call omits one. pub default_language: Option, + /// Cap, in characters, on each page an Exa contents call returns. `None` + /// asks for whole pages; values above the module's own per-page bound + /// (1,500) are lowered to it. + pub contents_max_chars: Option, } impl Default for ProviderConfig { fn default() -> Self { @@ -169,6 +173,7 @@ impl Default for ProviderConfig { max_results: None, timeout_secs: None, default_language: None, + contents_max_chars: None, } } } @@ -185,6 +190,7 @@ impl fmt::Debug for ProviderConfig { .field("max_results", &self.max_results) .field("timeout_secs", &self.timeout_secs) .field("default_language", &self.default_language) + .field("contents_max_chars", &self.contents_max_chars) .finish() } } diff --git a/crates/tinysearch/src/provider/direct/exa.rs b/crates/tinysearch/src/provider/direct/exa.rs index 2880e0e..0a293a9 100644 --- a/crates/tinysearch/src/provider/direct/exa.rs +++ b/crates/tinysearch/src/provider/direct/exa.rs @@ -37,7 +37,13 @@ pub(super) fn exa_body(request: &ExecuteToolRequest) -> Result<(&'static str, Va "/findSimilar", json!({"url":required_string(args,"url")?,"numResults":count(args,"max_results")}), ), - "exa_get_contents" => ("/contents", json!({"urls":urls(args)?,"text":true})), + "exa_get_contents" => { + let text = match args.get("max_characters").and_then(Value::as_u64) { + Some(max) if max > 0 => json!({"maxCharacters":max}), + _ => json!(true), + }; + ("/contents", json!({"urls":urls(args)?,"text":text})) + } "exa_answer" => { let mut body = json!({"query":required_string(args,"query")?}); if args.get("include_text") == Some(&Value::Bool(true)) { diff --git a/crates/tinysearch/src/provider/direct/exa_tests.rs b/crates/tinysearch/src/provider/direct/exa_tests.rs index 76e286e..76ba113 100644 --- a/crates/tinysearch/src/provider/direct/exa_tests.rs +++ b/crates/tinysearch/src/provider/direct/exa_tests.rs @@ -70,3 +70,38 @@ fn only_search_takes_a_type() -> Result<()> { assert!(body.get("type").is_none()); Ok(()) } + +fn contents(arguments: Value) -> ExecuteToolRequest { + ExecuteToolRequest { + name: "exa_get_contents".into(), + arguments, + } +} + +#[test] +fn contents_ask_for_whole_pages_unless_capped() -> Result<()> { + let (_, body) = exa_body(&contents(json!({"urls":["https://a.example"]})))?; + assert_eq!(body["text"], true); + Ok(()) +} + +#[test] +fn contents_cap_each_page_at_max_characters() -> Result<()> { + let (path, body) = exa_body(&contents( + json!({"urls":["https://a.example"],"max_characters":1500}), + ))?; + assert_eq!(path, "/contents"); + assert_eq!(body["text"], json!({"maxCharacters":1500})); + Ok(()) +} + +#[test] +fn a_zero_or_malformed_cap_is_ignored() -> Result<()> { + for cap in [json!(0), json!("many"), json!(-5)] { + let (_, body) = exa_body(&contents( + json!({"urls":["https://a.example"],"max_characters":cap}), + ))?; + assert_eq!(body["text"], true); + } + Ok(()) +} diff --git a/crates/tinysearch/src/provider/direct/mod.rs b/crates/tinysearch/src/provider/direct/mod.rs index ce22999..21c380c 100644 --- a/crates/tinysearch/src/provider/direct/mod.rs +++ b/crates/tinysearch/src/provider/direct/mod.rs @@ -170,6 +170,22 @@ fn image_asset(source_url: &str, image_url: &str, title: Option<&str>) -> Option Some(asset) } +/// Adds the configured per-page cap to an Exa contents call that set none. +/// +/// The cap rides in the call's `max_characters` argument, which +/// [`exa_body`] turns into Exa's `text.maxCharacters`. A cap above what +/// normalization keeps per page is lowered to that bound. +pub(super) fn apply_contents_cap(config: &ProviderConfig, request: &mut ExecuteToolRequest) { + if request.name == "exa_get_contents" + && request.arguments.get("max_characters").is_none() + && let Some(cap) = config.contents_max_chars + && let Some(args) = request.arguments.as_object_mut() + { + let bound = u64::try_from(super::MAX_PAGE_CHARS).unwrap_or(u64::MAX); + args.insert("max_characters".into(), Value::from(cap.clamp(1, bound))); + } +} + pub(super) fn prepare( provider: &str, config: &ProviderConfig, @@ -177,6 +193,7 @@ pub(super) fn prepare( key: &str, ) -> Result { let mut request = request.clone(); + apply_contents_cap(config, &mut request); let count_field = if provider == "brave" { "count" } else { diff --git a/crates/tinysearch/src/provider/mod.rs b/crates/tinysearch/src/provider/mod.rs index ae5073b..ba37f1c 100644 --- a/crates/tinysearch/src/provider/mod.rs +++ b/crates/tinysearch/src/provider/mod.rs @@ -16,6 +16,10 @@ const DEFAULT_GEMINI_THINKING_LEVEL: &str = "low"; const MAX_RESULTS: usize = 20; const MAX_CITATIONS: usize = 40; const MAX_ANSWER_CHARS: usize = 12_000; +const MAX_SNIPPET_CHARS: usize = 1_200; +/// The most text kept per page by a contents tool, and the most a configured +/// [`ProviderConfig::contents_max_chars`] asks a provider for. +const MAX_PAGE_CHARS: usize = 1_500; const MAX_BODY_BYTES: u64 = 2_000_000; /// Upper bound on how many Gemini grounding chunks/support entries are ever /// examined when selecting citations, independent of `MAX_CITATIONS`. Keeps @@ -118,7 +122,11 @@ impl BuiltinProvider { role: Option, ) -> Result { let (path, body) = match self.name { - "exa" if config.route == ProviderRoute::Backend => exa_request(request)?, + "exa" if config.route == ProviderRoute::Backend => { + let mut request = request.clone(); + direct::apply_contents_cap(config, &mut request); + exa_request(&request)? + } "exa" | "parallel" | "brave" | "querit" | "tavily" | "seltz" | "searxng" | "tinyfish" | "keenable" => { return direct::run(&self.client, self.name, config, request).await; diff --git a/crates/tinysearch/src/provider/mod_tests.rs b/crates/tinysearch/src/provider/mod_tests.rs index 6457442..3d70025 100644 --- a/crates/tinysearch/src/provider/mod_tests.rs +++ b/crates/tinysearch/src/provider/mod_tests.rs @@ -1155,6 +1155,92 @@ async fn exa_direct_operations_map_and_normalize() -> TestResult<()> { Ok(()) } +#[tokio::test] +async fn configured_contents_cap_bounds_pages_end_to_end() -> TestResult<()> { + let page = "p".repeat(4000); + let (url, server) = mock( + 200, + json!({"results":[{"url":"https://page.test","title":"Page","text":page}]}), + ) + .await?; + let response = direct::run( + &Client::new(), + "exa", + &ProviderConfig { + base_url: Some(url), + credential: Some("exa-secret".into()), + contents_max_chars: Some(1500), + ..ProviderConfig::default() + }, + &request("exa_get_contents", json!({"urls":["https://page.test"]})), + ) + .await?; + let sent = server.await??; + assert!(sent.contains("\"text\":{\"maxCharacters\":1500}")); + let snippet = response.results[0].snippet.as_deref().ok_or("no snippet")?; + assert_eq!(snippet.chars().count(), 1500); + Ok(()) +} + +#[test] +fn contents_cap_is_kept_within_what_normalization_keeps() { + let mut contents = request("exa_get_contents", json!({"urls":["https://a"]})); + let config = ProviderConfig { + contents_max_chars: Some(50_000), + ..ProviderConfig::default() + }; + direct::apply_contents_cap(&config, &mut contents); + assert_eq!(contents.arguments["max_characters"], 1500); +} + +#[test] +fn contents_cap_applies_to_the_backend_route_and_only_to_contents() -> TestResult<()> { + let config = ProviderConfig { + contents_max_chars: Some(800), + ..ProviderConfig::default() + }; + let mut contents = request("exa_get_contents", json!({"urls":["https://a"]})); + direct::apply_contents_cap(&config, &mut contents); + assert_eq!( + exa_request(&contents)?.1, + json!({"urls":["https://a"],"text":{"maxCharacters":800}}) + ); + let mut search = request("exa_search", json!({"query":"q"})); + direct::apply_contents_cap(&config, &mut search); + assert!(search.arguments.get("max_characters").is_none()); + let mut chosen = request( + "exa_get_contents", + json!({"urls":["https://a"],"max_characters":300}), + ); + direct::apply_contents_cap(&config, &mut chosen); + assert_eq!(chosen.arguments["max_characters"], 300); + let mut unset = request("exa_get_contents", json!({"urls":["https://a"]})); + direct::apply_contents_cap(&ProviderConfig::default(), &mut unset); + assert!(unset.arguments.get("max_characters").is_none()); + Ok(()) +} + +#[test] +fn contents_snippets_keep_the_page_cap_but_search_snippets_stay_bounded() { + let value = json!({"results":[{"url":"https://a","text":"z".repeat(3000)}]}); + let page = normalize("exa", "exa_get_contents", &value); + assert_eq!( + page.results[0] + .snippet + .as_deref() + .map(|s| s.chars().count()), + Some(1500) + ); + let search = normalize("exa", "exa_search", &value); + assert_eq!( + search.results[0] + .snippet + .as_deref() + .map(|s| s.chars().count()), + Some(1200) + ); +} + #[tokio::test] async fn brave_direct_operations_map_and_normalize() -> TestResult<()> { for (tool, path, payload) in [ diff --git a/crates/tinysearch/src/provider/normalize.rs b/crates/tinysearch/src/provider/normalize.rs index a44077d..eef7836 100644 --- a/crates/tinysearch/src/provider/normalize.rs +++ b/crates/tinysearch/src/provider/normalize.rs @@ -1,7 +1,11 @@ //! Bounded normalization of provider payloads. -use super::{MAX_ANSWER_CHARS, MAX_CITATIONS, MAX_GROUNDING_CHUNKS, MAX_RESULTS}; -use crate::{Citation, ExecuteToolResponse, SearchResult, SearchStatus}; +use super::{ + MAX_ANSWER_CHARS, MAX_CITATIONS, MAX_GROUNDING_CHUNKS, MAX_PAGE_CHARS, MAX_RESULTS, + MAX_SNIPPET_CHARS, +}; +use crate::{Citation, ExecuteToolResponse, Role, SearchResult, SearchStatus}; use serde_json::{Map, Value}; +use tinysearch_bus::role_provider_tool; fn clipped(text: &str, max: usize) -> String { text.chars().take(max).collect() @@ -12,7 +16,7 @@ fn get_text(value: &Value, field: &str) -> Option { .and_then(Value::as_str) .map(|s| clipped(s, MAX_ANSWER_CHARS)) } -fn add_result(results: &mut Vec, item: &Value) { +fn add_result(results: &mut Vec, item: &Value, snippet_chars: usize) { if results.len() >= MAX_RESULTS { return; } @@ -45,7 +49,7 @@ fn add_result(results: &mut Vec, item: &Value) { results.push(SearchResult { title: clipped(title, 300), url: clipped(url, 2048), - snippet: snippet.map(|s| clipped(s, 1200)), + snippet: snippet.map(|s| clipped(s, snippet_chars)), published: item .get("published_date") .or_else(|| item.get("publish_date")) @@ -208,9 +212,15 @@ fn add_grounding_citations(citations: &mut Vec, value: &Value) { pub(super) fn normalize(provider: &str, tool: &str, value: &Value) -> ExecuteToolResponse { let mut results = Vec::new(); let mut citations = Vec::new(); + // A page-contents tool keeps up to a page's worth of text per result. + let snippet_chars = if role_provider_tool(Role::Contents, provider) == Some(tool) { + MAX_PAGE_CHARS + } else { + MAX_SNIPPET_CHARS + }; if let Some(items) = value.get("results").and_then(Value::as_array) { for item in items { - add_result(&mut results, item); + add_result(&mut results, item, snippet_chars); if results.len() == MAX_RESULTS { break; }