From 111a95268ee7848967093d3081bff221e8d9ea07 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 8 Nov 2026 11:48:12 +0300 Subject: [PATCH 1/3] fix(render): keep default output, compare source hosts case-insensitively, budget contents answer Only Exa contents keep the page-sized snippet; other providers keep the 1,200-char snippet. Compact sources compare HTTP scheme/host case-insensitively. A contents response's answer (a copy of the first page) is left out under non-default render options. Co-authored-by: Medulla --- crates/tinysearch-bus/src/render/mod.rs | 44 ++++++++++++++++--- crates/tinysearch-bus/src/render/mod_tests.rs | 44 +++++++++++++++++++ crates/tinysearch/src/provider/mod_tests.rs | 15 +++++++ crates/tinysearch/src/provider/normalize.rs | 15 ++++--- 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/crates/tinysearch-bus/src/render/mod.rs b/crates/tinysearch-bus/src/render/mod.rs index 6ffffd4..718ad42 100644 --- a/crates/tinysearch-bus/src/render/mod.rs +++ b/crates/tinysearch-bus/src/render/mod.rs @@ -117,6 +117,40 @@ fn result_images(response: &ExecuteToolResponse) -> Vec> { .collect() } +/// A URL as compared for repeats: the scheme and host of an HTTP(S) URL are +/// case-insensitive, the rest is not. +fn url_key(url: &str) -> String { + let url = url.trim(); + let Some(split) = url.find("://") else { + return url.to_string(); + }; + let (scheme, rest) = url.split_at(split); + if !scheme.eq_ignore_ascii_case("http") && !scheme.eq_ignore_ascii_case("https") { + return url.to_string(); + } + let rest = &rest[3..]; + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(end); + format!( + "{}://{}{tail}", + scheme.to_ascii_lowercase(), + authority.to_ascii_lowercase() + ) +} + +/// The answer text to print. A page-contents response may carry the first +/// page as its answer as well as in its result snippet; under non-default +/// options that copy would bypass the per-page budget, so it is left out when +/// the page is listed. Answer-role responses are never touched, and the +/// default options keep the original output. +fn answer_text<'a>(response: &'a ExecuteToolResponse, options: &RenderOptions) -> Option<&'a str> { + let answer = non_empty(response.answer.as_deref())?; + let duplicates_page = response.role == Some(Role::Contents) + && !response.results.is_empty() + && *options != RenderOptions::default(); + (!duplicates_page).then_some(answer) +} + /// The citations to list under `Sources`, with their 1-based numbers. /// /// All of them unless `options.compact_sources` is set; then only those that @@ -134,14 +168,14 @@ fn sources<'a>( if !options.compact_sources { return numbered.collect(); } - let shown: HashSet<&str> = response + let shown: HashSet = response .results .iter() .take(max_results) - .map(|result| result.url.trim()) + .map(|result| url_key(&result.url)) .collect(); numbered - .filter(|(_, citation)| !shown.contains(citation.url.trim())) + .filter(|(_, citation)| !shown.contains(&url_key(&citation.url))) .take(max_results) .collect() } @@ -162,7 +196,7 @@ pub fn render_text_with( ) -> String { let excerpt_chars = options.excerpt_for(response.role); let mut lines = vec![heading(response, subject)]; - if let Some(answer) = non_empty(response.answer.as_deref()) { + if let Some(answer) = answer_text(response, options) { lines.push(String::new()); lines.push(answer.to_string()); } @@ -219,7 +253,7 @@ pub fn render_markdown_with( ) -> String { let excerpt_chars = options.excerpt_for(response.role); let mut out = format!("# {}\n", heading(response, subject)); - if let Some(answer) = non_empty(response.answer.as_deref()) { + if let Some(answer) = answer_text(response, options) { let _ = write!(out, "\n{answer}\n"); } let images = result_images(response); diff --git a/crates/tinysearch-bus/src/render/mod_tests.rs b/crates/tinysearch-bus/src/render/mod_tests.rs index ceb346a..c2fb0b3 100644 --- a/crates/tinysearch-bus/src/render/mod_tests.rs +++ b/crates/tinysearch-bus/src/render/mod_tests.rs @@ -287,6 +287,50 @@ fn contents_pages_are_bounded_by_the_contents_cap() { assert!(!text.contains(&"y".repeat(201))); } +#[test] +fn compact_matches_repeated_sources_by_case_insensitive_host_only() { + let mut r = listed(1); + r.results[0].url = "https://Example.COM/Path".into(); + r.citations = vec![ + Citation { + url: "HTTPS://example.com/Path".into(), + title: None, + }, + Citation { + url: "https://example.com/path".into(), + title: None, + }, + ]; + let text = render_text_with(&r, "q", 5, &RenderOptions::compact()); + assert!(!text.contains("[1]")); + assert!(text.contains("[2] https://example.com/path")); +} + +#[test] +fn contents_answer_copy_of_the_page_is_left_out_under_a_budget_only() { + let mut page = response(Some(Role::Contents)); + page.provider = "tinyfish".into(); + page.results[0].snippet = Some("y".repeat(4000)); + page.answer = Some("y".repeat(4000)); + let narrow = RenderOptions { + contents_chars: 200, + ..RenderOptions::default() + }; + for out in [ + render_text_with(&page, "u", 5, &narrow), + render_markdown_with(&page, "u", 5, &narrow), + ] { + assert!(!out.contains(&"y".repeat(201))); + } + // Default output is unchanged. + assert!(render_text(&page, "u", 5).contains(&"y".repeat(4000))); + // Answer-role responses are never trimmed. + let mut answer = answer_fixture(); + answer.answer = Some("a".repeat(4000)); + let out = render_text_with(&answer, "q", 5, &RenderOptions::compact()); + assert!(out.contains(&"a".repeat(4000))); +} + #[test] fn the_default_contents_cap_matches_todays_output() { let mut page = response(Some(Role::Contents)); diff --git a/crates/tinysearch/src/provider/mod_tests.rs b/crates/tinysearch/src/provider/mod_tests.rs index 3d70025..9aa18dc 100644 --- a/crates/tinysearch/src/provider/mod_tests.rs +++ b/crates/tinysearch/src/provider/mod_tests.rs @@ -1241,6 +1241,21 @@ fn contents_snippets_keep_the_page_cap_but_search_snippets_stay_bounded() { ); } +#[test] +fn non_exa_contents_snippets_keep_the_search_snippet_bound() { + let value = json!({"results":[{"url":"https://a","text":"z".repeat(3000)}]}); + for (provider, tool) in [("keenable", "keenable_fetch"), ("tavily", "tavily_extract")] { + let page = normalize(provider, tool, &value); + assert_eq!( + page.results[0] + .snippet + .as_deref() + .map(|s| s.chars().count()), + Some(1200) + ); + } +} + #[tokio::test] async fn brave_direct_operations_map_and_normalize() -> TestResult<()> { for (tool, path, payload) in [ diff --git a/crates/tinysearch/src/provider/normalize.rs b/crates/tinysearch/src/provider/normalize.rs index eef7836..8c6413c 100644 --- a/crates/tinysearch/src/provider/normalize.rs +++ b/crates/tinysearch/src/provider/normalize.rs @@ -212,12 +212,15 @@ fn add_grounding_citations(citations: &mut Vec, value: &Value) { pub(super) fn normalize(provider: &str, tool: &str, value: &Value) -> ExecuteToolResponse { let mut results = Vec::new(); let mut citations = Vec::new(); - // A page-contents tool keeps up to a page's worth of text per result. - let snippet_chars = if role_provider_tool(Role::Contents, provider) == Some(tool) { - MAX_PAGE_CHARS - } else { - MAX_SNIPPET_CHARS - }; + // Only Exa honours a per-page bound upstream (`contents_max_chars`), so + // only its contents tool keeps a page's worth of text per result; every + // other tool keeps today's snippet bound. + let snippet_chars = + if provider == "exa" && role_provider_tool(Role::Contents, provider) == Some(tool) { + MAX_PAGE_CHARS + } else { + MAX_SNIPPET_CHARS + }; if let Some(items) = value.get("results").and_then(Value::as_array) { for item in items { add_result(&mut results, item, snippet_chars); From 95bf478cb6d94bfcc8bf8f1292aed6fb43ce76be Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 8 Nov 2026 12:01:04 +0300 Subject: [PATCH 2/3] fix(bus): preserve userinfo case when deduplicating compact sources Only the scheme and host are case-insensitive; credentials in a URL's userinfo are compared verbatim. Co-authored-by: Medulla --- crates/tinysearch-bus/src/render/mod.rs | 10 +++++++--- crates/tinysearch-bus/src/render/mod_tests.rs | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/tinysearch-bus/src/render/mod.rs b/crates/tinysearch-bus/src/render/mod.rs index 718ad42..3ac82fb 100644 --- a/crates/tinysearch-bus/src/render/mod.rs +++ b/crates/tinysearch-bus/src/render/mod.rs @@ -118,7 +118,7 @@ fn result_images(response: &ExecuteToolResponse) -> Vec> { } /// A URL as compared for repeats: the scheme and host of an HTTP(S) URL are -/// case-insensitive, the rest is not. +/// case-insensitive, the rest (including any userinfo) is not. fn url_key(url: &str) -> String { let url = url.trim(); let Some(split) = url.find("://") else { @@ -131,10 +131,14 @@ fn url_key(url: &str) -> String { let rest = &rest[3..]; let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); let (authority, tail) = rest.split_at(end); + let (userinfo, host) = match authority.rfind('@') { + Some(at) => authority.split_at(at + 1), + None => ("", authority), + }; format!( - "{}://{}{tail}", + "{}://{userinfo}{}{tail}", scheme.to_ascii_lowercase(), - authority.to_ascii_lowercase() + host.to_ascii_lowercase() ) } diff --git a/crates/tinysearch-bus/src/render/mod_tests.rs b/crates/tinysearch-bus/src/render/mod_tests.rs index c2fb0b3..c10af1f 100644 --- a/crates/tinysearch-bus/src/render/mod_tests.rs +++ b/crates/tinysearch-bus/src/render/mod_tests.rs @@ -306,6 +306,25 @@ fn compact_matches_repeated_sources_by_case_insensitive_host_only() { assert!(text.contains("[2] https://example.com/path")); } +#[test] +fn compact_keeps_a_citation_whose_userinfo_differs_only_in_case() { + let mut r = listed(1); + r.results[0].url = "https://reader:TokenA@Example.com/Path".into(); + r.citations = vec![ + Citation { + url: "https://reader:TokenA@example.com/Path".into(), + title: None, + }, + Citation { + url: "https://reader:Tokena@example.com/Path".into(), + title: None, + }, + ]; + let text = render_text_with(&r, "q", 5, &RenderOptions::compact()); + assert!(!text.contains("[1]")); + assert!(text.contains("[2] https://reader:Tokena@example.com/Path")); +} + #[test] fn contents_answer_copy_of_the_page_is_left_out_under_a_budget_only() { let mut page = response(Some(Role::Contents)); From e020d950631fdefa54cc65461586e2088a86211b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 8 Nov 2026 12:23:00 +0300 Subject: [PATCH 3/3] fix(bus): keep contents answers that do not duplicate a listed page Under non-default options the answer is omitted only when it is a confirmed copy of a listed result's snippet; otherwise it is rendered clipped to the contents budget. This covers snippet-less pages and max_results of zero. Co-authored-by: Medulla --- crates/tinysearch-bus/src/render/mod.rs | 38 ++++++++++++------ crates/tinysearch-bus/src/render/mod_tests.rs | 39 +++++++++++++++++++ 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/crates/tinysearch-bus/src/render/mod.rs b/crates/tinysearch-bus/src/render/mod.rs index 3ac82fb..3370238 100644 --- a/crates/tinysearch-bus/src/render/mod.rs +++ b/crates/tinysearch-bus/src/render/mod.rs @@ -142,17 +142,31 @@ fn url_key(url: &str) -> String { ) } -/// The answer text to print. A page-contents response may carry the first -/// page as its answer as well as in its result snippet; under non-default -/// options that copy would bypass the per-page budget, so it is left out when -/// the page is listed. Answer-role responses are never touched, and the +/// The answer text to print. +/// +/// A page-contents response may carry the first page as its answer as well as +/// in its result snippet. Under non-default options that copy is left out, but +/// only when it is a confirmed copy of a page that is actually listed (one +/// text is a prefix of the other); any other contents answer is kept, clipped +/// to the per-page budget. Answer-role responses are never touched, and the /// default options keep the original output. -fn answer_text<'a>(response: &'a ExecuteToolResponse, options: &RenderOptions) -> Option<&'a str> { +fn answer_text( + response: &ExecuteToolResponse, + max_results: usize, + options: &RenderOptions, +) -> Option { let answer = non_empty(response.answer.as_deref())?; - let duplicates_page = response.role == Some(Role::Contents) - && !response.results.is_empty() - && *options != RenderOptions::default(); - (!duplicates_page).then_some(answer) + if response.role != Some(Role::Contents) || *options == RenderOptions::default() { + return Some(answer.to_string()); + } + let answer_trimmed = answer.trim(); + let copies_listed_page = response.results.iter().take(max_results).any(|result| { + non_empty(result.snippet.as_deref()).is_some_and(|snippet| { + let snippet = snippet.trim(); + answer_trimmed.starts_with(snippet) || snippet.starts_with(answer_trimmed) + }) + }); + (!copies_listed_page).then(|| truncate(answer, options.contents_chars, "…")) } /// The citations to list under `Sources`, with their 1-based numbers. @@ -200,9 +214,9 @@ pub fn render_text_with( ) -> String { let excerpt_chars = options.excerpt_for(response.role); let mut lines = vec![heading(response, subject)]; - if let Some(answer) = answer_text(response, options) { + if let Some(answer) = answer_text(response, max_results, options) { lines.push(String::new()); - lines.push(answer.to_string()); + lines.push(answer); } let images = result_images(response); for (index, item) in response.results.iter().take(max_results).enumerate() { @@ -257,7 +271,7 @@ pub fn render_markdown_with( ) -> String { let excerpt_chars = options.excerpt_for(response.role); let mut out = format!("# {}\n", heading(response, subject)); - if let Some(answer) = answer_text(response, options) { + if let Some(answer) = answer_text(response, max_results, options) { let _ = write!(out, "\n{answer}\n"); } let images = result_images(response); diff --git a/crates/tinysearch-bus/src/render/mod_tests.rs b/crates/tinysearch-bus/src/render/mod_tests.rs index c10af1f..554291f 100644 --- a/crates/tinysearch-bus/src/render/mod_tests.rs +++ b/crates/tinysearch-bus/src/render/mod_tests.rs @@ -325,6 +325,44 @@ fn compact_keeps_a_citation_whose_userinfo_differs_only_in_case() { assert!(text.contains("[2] https://reader:Tokena@example.com/Path")); } +#[test] +fn contents_answer_is_kept_when_no_snippet_duplicates_it() { + let mut page = response(Some(Role::Contents)); + page.results[0].snippet = None; + page.answer = Some("z".repeat(4000)); + let narrow = RenderOptions { + contents_chars: 200, + ..RenderOptions::default() + }; + for options in [RenderOptions::compact(), narrow] { + for out in [ + render_text_with(&page, "u", 5, &options), + render_markdown_with(&page, "u", 5, &options), + ] { + assert!(out.contains(&"z".repeat(options.contents_chars.min(200)))); + assert!(!out.contains(&"z".repeat(options.contents_chars + 1))); + } + } +} + +#[test] +fn contents_answer_is_kept_when_no_page_is_listed() { + let mut page = response(Some(Role::Contents)); + page.results[0].snippet = Some("y".repeat(300)); + page.answer = Some("y".repeat(300)); + let narrow = RenderOptions { + contents_chars: 100, + ..RenderOptions::default() + }; + for out in [ + render_text_with(&page, "u", 0, &narrow), + render_markdown_with(&page, "u", 0, &narrow), + ] { + assert_eq!(out.matches("y".repeat(100).as_str()).count(), 1); + assert!(!out.contains(&"y".repeat(101))); + } +} + #[test] fn contents_answer_copy_of_the_page_is_left_out_under_a_budget_only() { let mut page = response(Some(Role::Contents)); @@ -340,6 +378,7 @@ fn contents_answer_copy_of_the_page_is_left_out_under_a_budget_only() { render_markdown_with(&page, "u", 5, &narrow), ] { assert!(!out.contains(&"y".repeat(201))); + assert_eq!(out.matches("y".repeat(200).as_str()).count(), 1); } // Default output is unchanged. assert!(render_text(&page, "u", 5).contains(&"y".repeat(4000)));