facsim/config.yaml at main · highbaud/facsim · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# facsim configuration
# Quickest start:  facsim init   (interactive wizard writes this file for you)
# Then build:      facsim --config config.yaml

source:
  # The live site to mirror. All same-host links are followed.
  # NB: if the apex (example.com) 301-redirects to www and the sitemap entries
  # use www, set the canonical host to www here — otherwise the sitemap URLs
  # get filtered out as off-host.
  base_url: "https://www.example.com"
  # Optional extra start URLs (orphan pages) seeded into the crawl.
  seeds: []
  # Seed the crawl from the source's own sitemap (catches orphan pages
  # not reachable by following links). Follows one level of sitemap index.
  use_sitemap: true
  # Path to the site's sitemap. Common values:
  #   /sitemap.xml          plain sitemap (most sites)
  #   /sitemap_index.xml    sitemap index (WordPress + Yoast, etc.)
  sitemap_path: "/sitemap.xml"

cache:
  # Public base URL where the cache will be served (the subdomain).
  # Used for sitemap.xml loc entries and absolute links in index.html.
  base_url: "https://llm.example.com"
  # Brand name shown in the wordmark. Renders as "<brand> LLM Cache",
  # e.g. brand "Acme" -> "Acme LLM Cache".
  brand: "COMPANYNAME"
  # Optional path to an SVG logo (absolute, or relative to where you run the
  # build). When set and readable, the inlined SVG replaces the text wordmark
  # (it's sized to the wordmark height; its own colours are kept). Leave blank
  # to use the "<brand> LLM Cache" text wordmark.
  logo: ""
  # Human-readable label for the source site (index page / llms.txt header).
  site_name: "example.com"
  # Audiences advertised in llms.txt.
  audiences: ["GPT", "Claude", "Gemini", "Llama"]

crawl:
  max_pages: 500
  request_delay_seconds: 0.3
  timeout_seconds: 20
  user_agent: "facsim/1.0 (+content cache generator)"
  # Retry transient fetch failures (conn drops, 429/5xx) with exponential backoff.
  max_retries: 3
  retry_backoff_seconds: 0.5
  # Path prefixes to skip entirely (regex, matched against the path).
  exclude_patterns:
    - '\.(pdf|zip|jpg|jpeg|png|gif|svg|webp|mp4|mp3|css|js|ico|woff2?)$'
    - '^/wp-admin'
    - '^/wp-json'
    - '/feed/?$'
    - '\?'   # skip query-string URLs (usually duplicates/filters)
    # WordPress taxonomy/archive pages are thin index lists, not content —
    # keep the cache focused on real pages, posts, docs, and products.
    - '^/(category|tag|author)/'
    - '^/product-(category|tag)/'
    - '^/(product_cat|product_tag|post_tag|doc_category|extp_cat)/'

extract:
  # Skip pages whose extracted body is shorter than this (0 = keep everything).
  # Filters out JS-only shells and near-empty stubs so the corpus stays clean.
  min_content_words: 20
  # CSS selectors tried in order to locate the main content. First match wins.
  # Leave defaults for a generic site; tune once your site's markup is known.
  content_selectors:
    - "main"
    - "article"
    - "[role=main]"
    - "#content"
    - ".site-content"
    - ".entry-content"
  # Elements removed from the extracted content before conversion.
  strip_selectors:
    - "nav"
    - "header"
    - "footer"
    - "script"
    - "style"
    - "noscript"
    - "form"
    - ".cookie"
    - ".cookie-banner"
    - "[aria-hidden=true]"

enrich:
  # Optional LLM layer: a per-page TL;DR + FAQ (with FAQPage JSON-LD).
  # OFF by default — the generator runs fully without an API key or the
  # 'anthropic' package. When enabled, set ANTHROPIC_API_KEY in the environment
  # and `pip install anthropic`. If either is missing the build continues
  # without enrichment (it never fails). Results are cached per content hash,
  # so a rebuild only bills pages whose content actually changed.
  enabled: false
  summary: true          # generate the TL;DR
  faq: true              # generate the FAQ pairs
  faq_count: 4           # max Q/A pairs per page
  model: "claude-haiku-4-5-20251001"
  cache_dir: ".enrich-cache"
  max_input_chars: 12000 # truncate very long pages before the call

deploy:
  # Write a .htaccess into the output dir so Apache serves the machine files
  # with correct content types (.md as text/markdown, .jsonl as ndjson, UTF-8
  # everywhere) and sane caching. Harmless on other servers. The shipped Apache
  # vhost template carries the same directives for hosts without AllowOverride.
  emit_htaccess: true
  cache_html_seconds: 3600    # HTML / markdown / txt / jsonl / xml freshness
  cache_asset_seconds: 86400  # cache.css and other static assets

output:
  dir: "dist"