seo: IndexNow integration (Bing, Yandex, Seznam, Naver, DuckDuckGo)

IndexNow is an open protocol that lets a site push fresh URLs to participating search engines instantly, no rate limit. Bing, Yandex, Seznam, and Naver are participants; DuckDuckGo gets it through Bing. Google does NOT participate (use GSC URL Inspection for Google).

What landed:
- public/7399afbfb102ee7600c82332c40cdb26.txt: key file at site root, required by the protocol
- scripts/indexnow-submit.mjs: posts the live sitemap URLs (or specific URLs) to api.indexnow.org
- .github/workflows/indexnow.yml: re-submits on every push to src/content/, with a 90s grace period for Pages to redeploy first. Also supports manual workflow_dispatch with custom URL list

Tested locally with the full sitemap (42 URLs): 202 Accepted. Once this commit lands and Pages serves the key file, the workflow will re-submit on every content change.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ritabratamaiti

.github/workflows/indexnow.yml

@@ -0,0 +1,42 @@
+name: IndexNow
+
+# Submit fresh URLs to IndexNow (Bing, Yandex, Seznam, Naver, DuckDuckGo)
+# whenever docs content changes. Google is NOT an IndexNow participant; for
+# Google, use Search Console URL Inspection.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'src/content/**'
+      - 'public/sitemap.xml'
+  workflow_dispatch:
+    inputs:
+      urls:
+        description: 'Specific URLs (newline-separated). Empty = submit full sitemap.'
+        required: false
+        type: string
+
+jobs:
+  submit:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BrowyHQ/browyhq.github.io'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Wait for GitHub Pages to redeploy
+        # Give Pages a minute to publish the new sitemap before we submit.
+        run: sleep 90
+
+      - name: Submit to IndexNow
+        env:
+          MANUAL_URLS: ${{ github.event.inputs.urls }}
+        run: |
+          if [ -n "$MANUAL_URLS" ]; then
+            echo "$MANUAL_URLS" | xargs node scripts/indexnow-submit.mjs
+          else
+            node scripts/indexnow-submit.mjs
+          fi

public/7399afbfb102ee7600c82332c40cdb26.txt

@@ -0,0 +1 @@
+7399afbfb102ee7600c82332c40cdb26

scripts/indexnow-submit.mjs

@@ -0,0 +1,97 @@
+// scripts/indexnow-submit.mjs
+// Submit URLs from the live sitemap to IndexNow.
+// Hits api.indexnow.org once, which fans out to Bing, Yandex, Seznam, Naver,
+// and any other future IndexNow participants. Google is NOT a participant.
+//
+// Usage:
+//   node scripts/indexnow-submit.mjs                # submit all sitemap URLs
+//   node scripts/indexnow-submit.mjs <url> [<url>]  # submit specific URLs
+//
+// CI mode: triggered by .github/workflows/indexnow.yml on every push to main
+// that touches src/content/.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+
+const SITE = 'https://browyhq.github.io';
+const KEY = '7399afbfb102ee7600c82332c40cdb26';
+const KEY_LOCATION = `${SITE}/${KEY}.txt`;
+const ENDPOINT = 'https://api.indexnow.org/IndexNow';
+
+async function fetchSitemapURLs() {
+  const res = await fetch(`${SITE}/sitemap-0.xml`);
+  if (!res.ok) throw new Error(`sitemap fetch failed: ${res.status}`);
+  const xml = await res.text();
+  const urls = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+  return urls;
+}
+
+function chunk(arr, n) {
+  const out = [];
+  for (let i = 0; i < arr.length; i += n) out.push(arr.slice(i, i + n));
+  return out;
+}
+
+async function submit(urls) {
+  if (urls.length === 0) {
+    console.log('no urls to submit');
+    return;
+  }
+
+  // IndexNow caps a single POST at 10,000 URLs. We're well under that, but be defensive.
+  for (const batch of chunk(urls, 1000)) {
+    const body = {
+      host: new URL(SITE).host,
+      key: KEY,
+      keyLocation: KEY_LOCATION,
+      urlList: batch,
+    };
+
+    const res = await fetch(ENDPOINT, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json; charset=utf-8',
+        'Accept': 'application/json',
+      },
+      body: JSON.stringify(body),
+    });
+
+    const text = await res.text();
+    console.log(`POST ${ENDPOINT}  ${res.status} ${res.statusText}  (${batch.length} URL${batch.length === 1 ? '' : 's'})`);
+    if (text.trim()) console.log(`  response: ${text.slice(0, 300)}`);
+
+    // 200 OK = accepted. 202 = received but key still validating.
+    // 400/422 = bad payload. 403 = key file missing. 429 = throttled.
+    if (res.status !== 200 && res.status !== 202) {
+      throw new Error(`IndexNow returned ${res.status}`);
+    }
+  }
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  let urls = args.length > 0 ? args : await fetchSitemapURLs();
+
+  // De-dupe and keep only URLs on our host.
+  const host = new URL(SITE).host;
+  urls = [...new Set(urls)].filter((u) => {
+    try {
+      return new URL(u).host === host;
+    } catch {
+      return false;
+    }
+  });
+
+  console.log(`submitting ${urls.length} URL${urls.length === 1 ? '' : 's'} to IndexNow`);
+  await submit(urls);
+  console.log('done');
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});