feat: harden web search resilience

Author: JinFanZheng

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v1.8.0 - 2026-06-26
+
+### Added
+
+- In-process `web-search` result cache with `--no-cache` bypass.
+- DuckDuckGo Lite rate-limit detection with typed fallback semantics.
+
+### Improved
+
+- DuckDuckGo Lite rate-limit and anti-bot responses now retry briefly before returning an engine error or allowing auto/provider fallback.
+- Search resilience iteration plan for #2/#3, including why Brave/Mojeek are out of scope and Startpage remains a gated follow-up.
+
 ## v1.7.0 - 2026-06-16
 
 Feature work for local metrics and GUI observability.

README.md

@@ -144,6 +144,7 @@ web-tools web-search "latest AI news"
 web-tools web-search "AI latest developments" --locale en-US --limit 3
 web-tools web-search "golang readability" --include-domain github.com --exclude-domain reddit.com
 web-tools web-search "golang readability" --provider duckduckgo --json
+web-tools web-search "golang readability" --no-cache --json
 
 # Read a URL
 web-tools web-reader https://example.com/article
@@ -262,6 +263,10 @@ CLI flags override config defaults when provided. `--format=html` is only availa
 
 `web-reader --json` includes a `quality` object with extraction score, word count, minimum word threshold, fallback recommendation, and reasons. Sparse extraction warnings are written to stderr so stdout remains machine-consumable.
 
+`web-search` keeps a short in-process result cache to avoid duplicate backend requests in GUI and library-style usage. Use `--no-cache` to bypass it. The cache is not persisted to disk and does not store queries in metrics.
+
+DuckDuckGo Lite rate-limit or anti-bot responses are surfaced as structured engine errors and retried briefly before auto mode falls back to the next configured provider. Warnings stay on stderr so JSON stdout remains machine-consumable.
+
 ### Provider configuration
 
 `--provider` is the preferred selector for new integrations. `--engine` remains supported for compatibility with `auto`, `duckduckgo`, and `searxng`.

README.zh-CN.md

@@ -145,6 +145,7 @@ web-tools web-search "latest AI news"
 web-tools web-search "AI latest developments" --locale en-US --limit 3
 web-tools web-search "golang readability" --include-domain github.com --exclude-domain reddit.com
 web-tools web-search "golang readability" --provider duckduckgo --json
+web-tools web-search "golang readability" --no-cache --json
 
 # 读取 URL
 web-tools web-reader https://example.com/article
@@ -259,6 +260,10 @@ CLI 参数会覆盖配置默认值。`--format=html` 只在提取结果里真的
 
 `web-reader --json` 包含 `quality` 对象，包括提取评分、词数、最低词数阈值、是否建议 fallback 和原因。内容稀疏 warning 会写到 stderr，stdout 保持机器可读。
 
+`web-search` 会保留短生命周期的进程内结果缓存，减少 GUI 或库调用场景里的重复后端请求。使用 `--no-cache` 可以绕过缓存。缓存不会持久化到磁盘，metrics 也不会记录 query。
+
+DuckDuckGo Lite 的限流或反爬响应会返回结构化 engine error，并在 auto 模式 fallback 到下一个 provider 前做短重试。warning 只写 stderr，JSON stdout 保持机器可读。
+
 ### Provider 配置
 
 `--provider` 是新集成的推荐入口。`--engine` 继续保留，用于兼容 `auto`、`duckduckgo`、`searxng`。

cmd/web-search/main.go

@@ -23,6 +23,7 @@ func Cmd() *cobra.Command {
 		flagLocale         string
 		flagCat            string
 		flagTime           string
+		flagNoCache        bool
 		flagIncludeDomains []string
 		flagExcludeDomains []string
 	)
@@ -40,7 +41,7 @@ instance (opt-in, requires Docker). Zero cost, no API keys.`,
   web-tools web-search "climate change 2026" --time-range year --json`,
 		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			run(cmd, args[0], flagJSON, flagOutput, flagLimit, flagEngine, flagProvider, flagLocale, flagCat, flagTime, flagIncludeDomains, flagExcludeDomains)
+			run(cmd, args[0], flagJSON, flagOutput, flagLimit, flagEngine, flagProvider, flagLocale, flagCat, flagTime, flagNoCache, flagIncludeDomains, flagExcludeDomains)
 		},
 	}
 
@@ -52,13 +53,14 @@ instance (opt-in, requires Docker). Zero cost, no API keys.`,
 	cmd.Flags().StringVar(&flagLocale, "locale", "auto", "Language preference (zh-CN, en-US, auto)")
 	cmd.Flags().StringVar(&flagCat, "category", "general", "Search category: general / images / news / videos / files")
 	cmd.Flags().StringVar(&flagTime, "time-range", "any", "Time range: any / day / week / month / year")
+	cmd.Flags().BoolVar(&flagNoCache, "no-cache", false, "Bypass in-process search result cache")
 	cmd.Flags().StringSliceVar(&flagIncludeDomains, "include-domain", nil, "Only include results from domain(s); repeat or comma-separate")
 	cmd.Flags().StringSliceVar(&flagExcludeDomains, "exclude-domain", nil, "Exclude results from domain(s); repeat or comma-separate")
 
 	return cmd
 }
 
-func run(cmd *cobra.Command, query string, flagJSON bool, flagOutput string, flagLimit int, flagEngine string, flagProvider string, flagLocale string, flagCategory string, flagTimeRange string, flagIncludeDomains []string, flagExcludeDomains []string) {
+func run(cmd *cobra.Command, query string, flagJSON bool, flagOutput string, flagLimit int, flagEngine string, flagProvider string, flagLocale string, flagCategory string, flagTimeRange string, flagNoCache bool, flagIncludeDomains []string, flagExcludeDomains []string) {
 	start := time.Now()
 	var metric metrics.Event
 	var runErr error
@@ -77,7 +79,7 @@ func run(cmd *cobra.Command, query string, flagJSON bool, flagOutput string, fla
 	}
 	s := search.NewSearchWithConfig(*cfg)
 
-	opts, err := buildSearchOptions(cmd, flagLimit, flagEngine, flagProvider, flagLocale, flagCategory, flagTimeRange, flagIncludeDomains, flagExcludeDomains)
+	opts, err := buildSearchOptions(cmd, flagLimit, flagEngine, flagProvider, flagLocale, flagCategory, flagTimeRange, flagNoCache, flagIncludeDomains, flagExcludeDomains)
 	if err != nil {
 		runErr = err
 		apperrors.HandleError(runErr)
@@ -121,7 +123,7 @@ func firstNonEmpty(values ...string) string {
 	return ""
 }
 
-func buildSearchOptions(cmd *cobra.Command, flagLimit int, flagEngine string, flagProvider string, flagLocale string, flagCategory string, flagTimeRange string, flagIncludeDomains []string, flagExcludeDomains []string) (search.SearchOptions, error) {
+func buildSearchOptions(cmd *cobra.Command, flagLimit int, flagEngine string, flagProvider string, flagLocale string, flagCategory string, flagTimeRange string, flagNoCache bool, flagIncludeDomains []string, flagExcludeDomains []string) (search.SearchOptions, error) {
 	var opts search.SearchOptions
 	engineChanged := cmd.Flags().Changed("engine")
 	providerChanged := cmd.Flags().Changed("provider")
@@ -150,6 +152,9 @@ func buildSearchOptions(cmd *cobra.Command, flagLimit int, flagEngine string, fl
 	if cmd.Flags().Changed("time-range") {
 		opts.TimeRange = flagTimeRange
 	}
+	if cmd.Flags().Changed("no-cache") {
+		opts.NoCache = flagNoCache
+	}
 	if cmd.Flags().Changed("include-domain") {
 		opts.IncludeDomains = normalizeDomainFlags(flagIncludeDomains)
 	}

cmd/web-search/main_test.go

@@ -20,7 +20,7 @@ func TestLoadSearchConfig_UsesEnvOverrideForSearXNGURL(t *testing.T) {
 func TestBuildSearchOptions_OmittedFlagsStayZero(t *testing.T) {
 	cmd := Cmd()
 
-	opts, err := buildSearchOptions(cmd, 5, "auto", "auto", "auto", "general", "any", nil, nil)
+	opts, err := buildSearchOptions(cmd, 5, "auto", "auto", "auto", "general", "any", false, nil, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, search.SearchOptions{}, opts)
@@ -34,10 +34,11 @@ func TestBuildSearchOptions_ExplicitFlagsOverride(t *testing.T) {
 	require.NoError(t, cmd.Flags().Set("locale", "en-US"))
 	require.NoError(t, cmd.Flags().Set("category", "news"))
 	require.NoError(t, cmd.Flags().Set("time-range", "week"))
+	require.NoError(t, cmd.Flags().Set("no-cache", "true"))
 	require.NoError(t, cmd.Flags().Set("include-domain", "example.com,docs.example.com"))
 	require.NoError(t, cmd.Flags().Set("exclude-domain", "spam.example"))
 
-	opts, err := buildSearchOptions(cmd, 2, "searxng", "searxng", "en-US", "news", "week", []string{"example.com", "docs.example.com"}, []string{"spam.example"})
+	opts, err := buildSearchOptions(cmd, 2, "searxng", "searxng", "en-US", "news", "week", true, []string{"example.com", "docs.example.com"}, []string{"spam.example"})
 
 	require.NoError(t, err)
 	assert.Equal(t, 2, opts.Limit)
@@ -46,6 +47,7 @@ func TestBuildSearchOptions_ExplicitFlagsOverride(t *testing.T) {
 	assert.Equal(t, "en-US", opts.Locale)
 	assert.Equal(t, "news", opts.Category)
 	assert.Equal(t, "week", opts.TimeRange)
+	assert.True(t, opts.NoCache)
 	assert.Equal(t, []string{"example.com", "docs.example.com"}, opts.IncludeDomains)
 	assert.Equal(t, []string{"spam.example"}, opts.ExcludeDomains)
 }
@@ -54,7 +56,7 @@ func TestBuildSearchOptions_ProviderWithoutEngine(t *testing.T) {
 	cmd := Cmd()
 	require.NoError(t, cmd.Flags().Set("provider", "duckduckgo"))
 
-	opts, err := buildSearchOptions(cmd, 5, "auto", "duckduckgo", "auto", "general", "any", nil, nil)
+	opts, err := buildSearchOptions(cmd, 5, "auto", "duckduckgo", "auto", "general", "any", false, nil, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, "duckduckgo", opts.Provider)
@@ -66,7 +68,7 @@ func TestBuildSearchOptions_ConflictingEngineAndProvider(t *testing.T) {
 	require.NoError(t, cmd.Flags().Set("engine", "searxng"))
 	require.NoError(t, cmd.Flags().Set("provider", "duckduckgo"))
 
-	_, err := buildSearchOptions(cmd, 5, "searxng", "duckduckgo", "auto", "general", "any", nil, nil)
+	_, err := buildSearchOptions(cmd, 5, "searxng", "duckduckgo", "auto", "general", "any", false, nil, nil)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "conflicting provider flags")