feat: improve extractor robustness and add test workflow

harden backend fetch safety and URL normalization
add inline SVG extraction + frontend data URL asset support
expand image candidate discovery from head metadata
add fixture tests and corpus fetch/report/test tooling
document new commands and config

Author: jtbrough

.gitignore

@@ -21,3 +21,8 @@ pnpm-debug.log*
 coverage/
 dist/
 tmp/
+
+# Generated corpus snapshots/reports
+tests/fixtures/corpus/*.html
+tests/fixtures/corpus/index.tsv
+tests/fixtures/corpus/report.tsv

Justfile

@@ -1,6 +1,18 @@
 run:
 	node server.js
 
+test:
+	node --test tests/*.test.js
+
+corpus-fetch:
+	sh scripts/fetch-corpus.sh
+
+test-corpus:
+	node --test tests/corpus.extract.test.js
+
+corpus-report:
+	node scripts/corpus-report.js
+
 docker-build:
 	docker build --pull --no-cache -t imagefetch:latest .
 

README.md

@@ -80,6 +80,10 @@ just --list
 Current commands:
 
 - `just run` - run Node server directly
+- `just test` - run fixture-based extraction tests
+- `just corpus-fetch` - fetch `tests/corpus/urls.txt` into local corpus fixtures
+- `just test-corpus` - run extraction tests against fetched corpus fixtures
+- `just corpus-report` - generate per-site detected-image report from fetched corpus
 - `just docker-build` - build latest container image
 - `just docker-run` - run hardened container
 - `just ci-lint` - lint GitHub Actions workflows
@@ -98,9 +102,25 @@ Environment variables:
 - `MAX_HTML_BYTES` (default `2097152`)
 - `MAX_ASSET_BYTES` (default `26214400`)
 - `MAX_REQUEST_TARGET_CHARS` (default `4096`)
+- `APK_UPGRADE_ON_START` (default `1`, set `0` to skip `apk update && apk upgrade` at container startup)
 
 App version is read from the `VERSION` file at server startup.
 
+## Corpus Workflow
+
+Use this to validate extraction behavior against a real-world URL corpus:
+
+1. Add target URLs to `tests/corpus/urls.txt` (one per line)
+2. Fetch snapshots:
+   - `just corpus-fetch`
+3. Run corpus tests:
+   - `just test-corpus`
+4. Generate a detection report:
+   - `just corpus-report`
+
+Fetched snapshots are stored in `tests/fixtures/corpus/` with an index manifest at `tests/fixtures/corpus/index.tsv`.
+Detection report is written to `tests/fixtures/corpus/report.tsv`.
+
 ## Security Notes
 
 - SSRF protections block localhost/private address targets

index.html

@@ -758,6 +758,19 @@ <h1 class="title">ImageFetch</h1>
       return { controller, clear: () => clearTimeout(timer) };
     }
 
+    function dataUrlToBlob(dataUrl) {
+      const str = String(dataUrl || '');
+      const match = str.match(/^data:([^;,]+)?(?:;base64)?,(.*)$/i);
+      if (!match) throw new Error('Invalid data URL');
+      const mime = match[1] || 'application/octet-stream';
+      const payload = match[2] || '';
+      const isBase64 = /;base64,/i.test(str.slice(0, str.indexOf(',') + 1));
+      const bytes = isBase64
+        ? Uint8Array.from(atob(payload), (ch) => ch.charCodeAt(0))
+        : Uint8Array.from(decodeURIComponent(payload), (ch) => ch.charCodeAt(0));
+      return new Blob([bytes], { type: mime });
+    }
+
     async function fetchExtraction(url, config) {
       const guard = withTimeout(config.timeoutMs, 'Page request');
       try {
@@ -781,6 +794,19 @@ <h1 class="title">ImageFetch</h1>
     }
 
     async function fetchAssetBlob(item, timeoutMs) {
+      if (String(item.url || '').startsWith('data:image/')) {
+        try {
+          const blob = dataUrlToBlob(item.url);
+          item.blob = blob;
+          item.mime = blob.type || 'image/svg+xml';
+          item.objectUrl = URL.createObjectURL(blob);
+        } catch (err) {
+          item.loadError = String(err.message || err);
+        }
+        item.dimensions = await detectDimensions(item.objectUrl || item.url);
+        return item;
+      }
+
       const guard = withTimeout(timeoutMs, `Asset request (${item.filename})`);
       try {
         const params = new URLSearchParams({ url: item.url, timeoutMs: String(timeoutMs) });

scripts/corpus-report.js

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+'use strict';
+
+const { readFileSync, writeFileSync, existsSync } = require('node:fs');
+const path = require('node:path');
+const { extractCandidates } = require('../server.js');
+
+const corpusDir = path.join(__dirname, '..', 'tests', 'fixtures', 'corpus');
+const indexPath = path.join(corpusDir, 'index.tsv');
+const outPath = path.join(corpusDir, 'report.tsv');
+
+function parseIndex(tsv) {
+  const lines = String(tsv || '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  if (lines.length < 2) return [];
+  const out = [];
+  for (let i = 1; i < lines.length; i += 1) {
+    const cols = lines[i].split('\t');
+    out.push({
+      id: cols[0] || '',
+      url: cols[1] || '',
+      file: cols[2] || '',
+      status: cols[3] || '',
+      detail: cols[4] || '',
+    });
+  }
+  return out;
+}
+
+if (!existsSync(indexPath)) {
+  console.error(`Missing corpus index: ${indexPath}`);
+  process.exit(1);
+}
+
+const rows = parseIndex(readFileSync(indexPath, 'utf8')).filter((r) => r.status === 'ok');
+if (rows.length === 0) {
+  console.error('No successful corpus rows. Run: just corpus-fetch');
+  process.exit(2);
+}
+
+const reportLines = ['id\turl\tgroup\tsourceType\tfilename\tassetUrl'];
+let totalAssets = 0;
+for (const row of rows) {
+  const fixturePath = path.join(corpusDir, row.file);
+  if (!existsSync(fixturePath)) continue;
+  const html = readFileSync(fixturePath, 'utf8');
+  const groups = extractCandidates(html, row.url, 3);
+  for (const group of groups) {
+    for (const item of group.items) {
+      totalAssets += 1;
+      reportLines.push([
+        row.id,
+        row.url,
+        group.key,
+        item.sourceType || '',
+        item.filename || '',
+        item.url || '',
+      ].join('\t'));
+    }
+  }
+}
+
+writeFileSync(outPath, `${reportLines.join('\n')}\n`, 'utf8');
+console.log(`Corpus report written: ${outPath}`);
+console.log(`Sites: ${rows.length}, Assets: ${totalAssets}`);

scripts/fetch-corpus.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+set -eu
+
+URLS_FILE="${1:-tests/corpus/urls.txt}"
+OUT_DIR="${2:-tests/fixtures/corpus}"
+INDEX_FILE="${OUT_DIR}/index.tsv"
+PRECHECK_HOST="${CORPUS_PRECHECK_HOST:-github.com}"
+
+if [ ! -f "${URLS_FILE}" ]; then
+  echo "URLs file not found: ${URLS_FILE}" >&2
+  exit 1
+fi
+
+if ! command -v curl >/dev/null 2>&1; then
+  echo "curl is required but not found in PATH" >&2
+  exit 1
+fi
+
+# Fail fast when command-line DNS/network is unavailable in this runtime.
+if ! node -e "require('node:dns').lookup('${PRECHECK_HOST}', (e) => process.exit(e ? 1 : 0))" >/dev/null 2>&1; then
+  echo "DNS precheck failed for ${PRECHECK_HOST}. This runtime cannot resolve external hosts." >&2
+  echo "Run corpus fetch in a network-enabled shell (outside restricted sandbox) and retry." >&2
+  exit 2
+fi
+
+mkdir -p "${OUT_DIR}"
+rm -f "${OUT_DIR}"/*.html
+printf "id\turl\tfile\tstatus\tdetail\n" > "${INDEX_FILE}"
+
+i=0
+while IFS= read -r line || [ -n "${line}" ]; do
+  url="$(printf "%s" "${line}" \
+    | tr -d '\r' \
+    | sed '1s/^\xEF\xBB\xBF//' \
+    | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \
+    | sed 's/^[-*][[:space:]]*//' \
+    | sed 's/[[:space:]]\+#.*$//' \
+    | sed 's/[;,][[:space:]]*$//')"
+  case "${url}" in
+    ""|\#*) continue ;;
+  esac
+
+  case "${url}" in
+    http://*|https://*) ;;
+    *) url="https://${url}" ;;
+  esac
+
+  i=$((i + 1))
+  id="$(printf "%03d" "${i}")"
+  host="$(printf "%s" "${url}" | sed -E 's#^[a-zA-Z]+://##; s#/.*$##; s#[:?&=]#_#g')"
+  [ -n "${host}" ] || host="site"
+  file="${id}-${host}.html"
+  out_path="${OUT_DIR}/${file}"
+
+  err_file="${OUT_DIR}/.${id}.curl.err"
+  if curl -fsSL \
+    --retry 2 \
+    --retry-all-errors \
+    --connect-timeout 10 \
+    --max-time 30 \
+    -H 'Accept: text/html,application/xhtml+xml' \
+    -H 'Accept-Language: en-US,en;q=0.9' \
+    -A 'ImageFetchCorpus/1.0 (+https://github.com/jtbrough/ImageFetch)' \
+    "${url}" \
+    -o "${out_path}" \
+    2>"${err_file}"; then
+    printf "%s\t%s\t%s\tok\t-\n" "${id}" "${url}" "${file}" >> "${INDEX_FILE}"
+    rm -f "${err_file}"
+    echo "ok   ${url} -> ${file}"
+  else
+    rc=$?
+    detail="$(tr '\n' ' ' < "${err_file}" | sed 's/[[:space:]]\+/ /g; s/^[[:space:]]*//; s/[[:space:]]*$//')"
+    rm -f "${err_file}"
+    [ -n "${detail}" ] || detail="curl_exit_${rc}"
+    rm -f "${out_path}"
+    printf "%s\t%s\t%s\tfetch_failed\t%s\n" "${id}" "${url}" "${file}" "${detail}" >> "${INDEX_FILE}"
+    echo "fail ${url} (${detail})" >&2
+  fi
+done < "${URLS_FILE}"
+
+echo "Wrote corpus index: ${INDEX_FILE}"