Comparison: normalize man page header, content-based tarball diff, adopt CI man page on normalized match

Author: danielrobbins

docs/release-steps.md

@@ -59,7 +59,10 @@ make release   # for first publication
 You will see:
 1. Local build presence check (or build via prerequisites).
 2. CI artifact fetch (MANDATORY). Failure to retrieve artifacts aborts; you must wait for the workflow to finish.
-3. sha256 comparison (local vs CI). If any differ, release ABORTS by default (no prompt) to enforce deterministic provenance.
+3. Normalized comparison phase:
+   * `keychain` – raw sha256 digest compare.
+   * `keychain.1` – compare raw hash; if different, re-compare with the Pod::Man auto-generated first line stripped. If normalized content matches, we ADOPT the CI man page and treat as match.
+   * `keychain-<version>.tar.gz` – unpack both tarballs; compare sorted file list and per-file sha256. If all internal files match, we treat the tarballs as equivalent even if gzip/tar metadata differ (timestamp, owner, compression). Any real content divergence aborts.
    - To force using local artifacts: `KEYCHAIN_FORCE_LOCAL=1 make release`
    - To adopt CI artifacts: `KEYCHAIN_ADOPT_CI=1 make release`
    (Use corresponding `... make release-refresh` for refresh mode.)

scripts/release-orchestrate.sh

@@ -35,21 +35,109 @@ echo "CI artifacts retrieved." >&2
 calc_sha256() { sha256sum "$1" | awk '{print $1}'; }
 
 diff_flag=0
-echo "Digest comparison (sha256):"
-for artifact in keychain-$VER.tar.gz keychain keychain.1; do
-  if [ -f "$CI_DIR/$artifact" ]; then
-    L=$(calc_sha256 "$artifact")
-    R=$(calc_sha256 "$CI_DIR/$artifact")
-    if [ "$L" = "$R" ]; then
-      printf '  %-20s %s (match)\n' "$artifact" "$L"
-    else
-      printf '  %-20s LOCAL %s != CI %s  *DIFF*\n' "$artifact" "$L" "$R"
-      diff_flag=1
+echo "Digest comparison (normalized where applicable):"
+
+normalize_man() {
+  # Strip the auto-generated Pod::Man first line so version differences don't cause false mismatch.
+  # Output to stdout.
+  sed '1{/^\\." Automatically generated by Pod::Man /d;}' "$1"
+}
+
+compare_tar_content() {
+  local local_tar=$1 ci_tar=$2
+  local tmp_local tmp_ci
+  tmp_local=$(mktemp -d)
+  tmp_ci=$(mktemp -d)
+  # Extract quietly
+  tar xzf "$local_tar" -C "$tmp_local" 2>/dev/null || return 2
+  tar xzf "$ci_tar" -C "$tmp_ci" 2>/dev/null || return 2
+  # Determine root (expect exactly one directory named keychain-$VER)
+  local root="keychain-$VER"
+  if [ ! -d "$tmp_local/$root" ] || [ ! -d "$tmp_ci/$root" ]; then
+    echo "  keychain-$VER.tar.gz: unexpected directory layout inside tar" >&2
+    return 3
+  fi
+  # List files (regular only) relative to root
+  local lf cf
+  lf=$( (cd "$tmp_local/$root" && find . -type f -print | LC_ALL=C sort) )
+  cf=$( (cd "$tmp_ci/$root" && find . -type f -print | LC_ALL=C sort) )
+  if [ "$lf" != "$cf" ]; then
+    echo "  keychain-$VER.tar.gz: file list differs" >&2
+    return 4
+  fi
+  # Hash each file
+  local mismatch=0
+  while IFS= read -r rel; do
+    [ -z "$rel" ] && continue
+    local h1 h2
+    h1=$(sha256sum "$tmp_local/$root/$rel" | awk '{print $1}')
+    h2=$(sha256sum "$tmp_ci/$root/$rel" | awk '{print $1}')
+    if [ "$h1" != "$h2" ]; then
+      echo "  keychain-$VER.tar.gz: content mismatch in $rel" >&2
+      mismatch=1
     fi
+  done <<EOF
+$lf
+EOF
+  if [ $mismatch -eq 0 ]; then
+    # Even if tarball blob hashes differ, content matches.
+    return 0
   else
+    return 5
+  fi
+}
+
+# Process artifacts with specialized logic
+for artifact in keychain keychain.1 keychain-$VER.tar.gz; do
+  if [ ! -f "$CI_DIR/$artifact" ]; then
     printf '  %-20s CI copy missing; comparison failed (abort)\n' "$artifact"
     diff_flag=1
+    continue
   fi
+  case "$artifact" in
+    keychain)
+      L=$(calc_sha256 "$artifact"); R=$(calc_sha256 "$CI_DIR/$artifact")
+      if [ "$L" = "$R" ]; then
+        printf '  %-20s %s (match)\n' "$artifact" "$L"
+      else
+        printf '  %-20s LOCAL %s != CI %s  *DIFF*\n' "$artifact" "$L" "$R"
+        diff_flag=1
+      fi
+      ;;
+    keychain.1)
+      # Direct hash first
+      L=$(calc_sha256 "$artifact"); R=$(calc_sha256 "$CI_DIR/$artifact")
+      if [ "$L" = "$R" ]; then
+        printf '  %-20s %s (match)\n' "$artifact" "$L"
+      else
+        # Normalize and compare ignoring Pod::Man header line.
+        if diff -u <(normalize_man "$artifact") <(normalize_man "$CI_DIR/$artifact") >/dev/null 2>&1; then
+          printf '  %-20s (normalized match ignoring Pod::Man header)\n' "$artifact"
+          # Adopt CI version so released man page reflects builder environment deterministically.
+          cp -f "$CI_DIR/$artifact" "$artifact"
+        else
+          printf '  %-20s LOCAL %s != CI %s  *DIFF* (content mismatch beyond header)\n' "$artifact" "$L" "$R"
+          diff_flag=1
+        fi
+      fi
+      ;;
+    keychain-$VER.tar.gz)
+      if compare_tar_content "$artifact" "$CI_DIR/$artifact"; then
+        # If tar blob hash matches display it; else note normalized match.
+        L=$(calc_sha256 "$artifact"); R=$(calc_sha256 "$CI_DIR/$artifact")
+        if [ "$L" = "$R" ]; then
+          printf '  %-20s %s (match)\n' "$artifact" "$L"
+        else
+          printf '  %-20s (content match; tar/gzip metadata differ)\n' "$artifact"
+          # Optional: adopt CI tarball for consistency (uncomment if desired)
+          # cp -f "$CI_DIR/$artifact" "$artifact"
+        fi
+      else
+        printf '  %-20s *CONTENT DIFF* (see above messages)\n' "$artifact"
+        diff_flag=1
+      fi
+      ;;
+  esac
 done
 
 if [ $diff_flag -ne 0 ]; then