Merge pull request #276 from RayforceDB/dev

v2.1.3

Author: singaraiona

.github/workflows/release.yml

@@ -101,9 +101,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # Distributables must be PORTABLE, not -march=native: a native binary
+        # built on a runner can SIGILL on a different/older CPU. x86-64-v3 =
+        # AVX2 baseline (~2013+), which keeps the engine's vectorized reductions
+        # fast. The macOS runner is the oldest Apple-Silicon class, so native
+        # there is effectively a safe floor (arm64 has no x86-style optional-ISA
+        # traps, and baselining to armv8-a would only lose M1 tuning).
         include:
           - os: ubuntu-latest
+            march: x86-64-v3
           - os: macos-latest
+            march: native
           # Windows is not build-ready yet (IOCP stub, unguarded POSIX in
           # main.c/heap.c, no Makefile path). Add once the port lands:
           # - os: windows-latest
@@ -114,8 +122,8 @@ jobs:
         with:
           ref: ${{ needs.prepare.outputs.sha }}
 
-      - name: Build release artifact
-        run: make dist RAY_VERSION=${{ needs.prepare.outputs.version }}
+      - name: Build release artifact (portable)
+        run: make dist RAY_VERSION=${{ needs.prepare.outputs.version }} RAY_MARCH=${{ matrix.march }}
 
       - name: Upload artifacts to release
         env:
@@ -126,6 +134,28 @@ jobs:
           gh release upload "v${{ needs.prepare.outputs.version }}" \
             dist/*.tar.gz dist/*.sha256 --clobber
 
+      # Package the just-built (already-portable) Linux binary as a .deb —
+      # `make dist` left ./rayforce in place, so no rebuild is needed.
+      - name: Build .deb (Linux)
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          RAY_VERSION: ${{ needs.prepare.outputs.version }}
+        run: |
+          set -euo pipefail
+          ver="$(curl -fsSL https://api.github.com/repos/goreleaser/nfpm/releases/latest | grep -oP '"tag_name":\s*"\K[^"]+')"
+          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/${ver}/nfpm_${ver#v}_amd64.deb" -o /tmp/nfpm.deb
+          sudo dpkg -i /tmp/nfpm.deb
+          mkdir -p dist
+          nfpm pkg --packager deb --config packaging/nfpm.yaml --target dist/
+          ls -l dist/*.deb
+
+      - name: Upload .deb to release (Linux)
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: gh release upload "v${{ needs.prepare.outputs.version }}" dist/*.deb --clobber
+
   publish:
     needs: [prepare, build]
     runs-on: ubuntu-latest
@@ -169,3 +199,40 @@ jobs:
             --data-urlencode "content=${content}")"
           echo "Zulip response: $resp"
           echo "$resp" | grep -q '"result": *"success"'
+
+  # Bump the Homebrew tap (RayforceDB/homebrew-tap → Formula/rayforce.rb) to this
+  # release. Best-effort: skipped when HOMEBREW_TAP_TOKEN isn't configured, and
+  # continue-on-error so a tap hiccup never reds an already-shipped release.
+  homebrew:
+    needs: [prepare, publish]
+    runs-on: ubuntu-latest
+    env:
+      HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          ref: ${{ needs.prepare.outputs.sha }}
+      - name: Update tap formula
+        if: ${{ env.HOMEBREW_TAP_TOKEN != '' }}
+        continue-on-error: true
+        env:
+          VERSION: ${{ needs.prepare.outputs.version }}
+          GH_REPO: ${{ github.repository }}
+          TAP_REPO: RayforceDB/homebrew-tap
+        run: |
+          set -euo pipefail
+          # Build-from-source formula: point at the GitHub source tarball for the
+          # tag and pin its sha256.
+          url="https://github.com/${GH_REPO}/archive/refs/tags/v${VERSION}.tar.gz"
+          sha="$(curl -fsSL "$url" | sha256sum | cut -d' ' -f1)"
+          tmp="$(mktemp -d)"
+          git clone --depth 1 "https://x-access-token:${HOMEBREW_TAP_TOKEN}@github.com/${TAP_REPO}.git" "$tmp"
+          mkdir -p "$tmp/Formula"
+          sed -e "s|__URL__|${url}|" -e "s|__SHA256__|${sha}|" \
+            packaging/homebrew-formula.rb.tmpl > "$tmp/Formula/rayforce.rb"
+          cd "$tmp"
+          git config user.name  "rayforce-release"
+          git config user.email "release@rayforcedb.com"
+          git add Formula/rayforce.rb
+          git commit -m "rayforce ${VERSION}" || { echo "formula unchanged — nothing to push"; exit 0; }
+          git push origin HEAD

Makefile

@@ -31,9 +31,16 @@ DEPFLAGS = -MMD -MP
 
 UNAME_S := $(shell uname -s)
 
-DEBUG_CFLAGS   = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=native -DDEBUG \
+# Target microarchitecture.  Default `native` = build for THIS machine (fastest;
+# right for local builds and the per-machine release tarballs).  Override for
+# REDISTRIBUTABLE packages (.deb) that must run on any CPU, e.g.
+# `make release RAY_MARCH=x86-64-v3` (AVX2 baseline, ~2013+) — a -march=native
+# binary handed to a different/older CPU dies with SIGILL.
+RAY_MARCH ?= native
+
+DEBUG_CFLAGS   = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=$(RAY_MARCH) -DDEBUG \
   -fsanitize=address,undefined -fno-omit-frame-pointer
-RELEASE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=native \
+RELEASE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=$(RAY_MARCH) \
   -funroll-loops -fomit-frame-pointer -fno-math-errno \
   -fassociative-math -ffp-contract=fast -fno-signed-zeros -fno-trapping-math
 # -fassociative-math: license to reorder FP additions/multiplications.
@@ -52,7 +59,7 @@ RELEASE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=native \
 # with the profile runtime, so we drop them; -O0 keeps line numbers
 # and avoids dead-code regions getting marked uncovered for the
 # wrong reason.  See `make coverage` below.
-COVERAGE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=native -DDEBUG \
+COVERAGE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=$(RAY_MARCH) -DDEBUG \
   -fno-omit-frame-pointer -fprofile-instr-generate -fcoverage-mapping
 COVERAGE_LDFLAGS = -fprofile-instr-generate -fcoverage-mapping
 

README.md

@@ -27,6 +27,26 @@ analytics and graph traversals share a single operation DAG, pass through a
 multi-pass optimizer, and execute as morsel-driven bytecode compiled at
 execution time. No malloc.
 
+## Install
+
+**Homebrew** (macOS & Linux):
+
+```bash
+brew install rayforcedb/tap/rayforce
+```
+
+**Debian / Ubuntu** (`.deb`, x86-64) — grab the `.deb` from the
+[latest release](https://github.com/RayforceDB/rayforce/releases/latest):
+
+```bash
+curl -LO https://github.com/RayforceDB/rayforce/releases/download/vX.Y.Z/rayforce_X.Y.Z_amd64.deb
+sudo dpkg -i rayforce_X.Y.Z_amd64.deb
+```
+
+**Prebuilt tarball** (Linux x86-64 / macOS arm64) is also attached to each
+release. Or build [from source](#quick-start) below — zero dependencies, just
+`make`.
+
 ## Quick Start
 
 ```bash

RELEASE.md

@@ -96,6 +96,34 @@ make dist RAY_VERSION=2.1.3
   is skipped, and a Zulip error never fails the release (`continue-on-error`).
   If Zulip rejects the post, the bot may need to be a *Generic bot* (not just an
   incoming-webhook) and subscribed to the Announcements channel.
+- **Optional — Homebrew tap**: create an empty repo **`RayforceDB/homebrew-tap`**
+  and add a repository secret **`HOMEBREW_TAP_TOKEN`** (a PAT — fine-grained or
+  classic — with **contents: write** on `homebrew-tap`; the default
+  `GITHUB_TOKEN` can't push to a *different* repo). The `homebrew` job then
+  rewrites `Formula/rayforce.rb` in the tap on every release. Without the secret
+  the job is skipped. Users install with `brew install rayforcedb/tap/rayforce`.
+
+## Packaging / install channels
+
+Each release publishes, in addition to the source:
+
+- **Tarballs** — `rayforce-X.Y.Z-linux-x86_64.tar.gz` and
+  `…-darwin-arm64.tar.gz` (+ `.sha256`). The Linux one is **portable**
+  (`RAY_MARCH=x86-64-v3`, AVX2/~2013+) so it can't SIGILL on a different/older
+  CPU. macOS stays `-march=native`: the runner is the oldest Apple-Silicon
+  class, so it's already a safe floor (arm64 has no x86-style optional-ISA
+  traps, and baselining to `armv8-a` would only cost M1 tuning).
+- **`.deb`** (`rayforce_X.Y.Z_amd64.deb`) — the same portable Linux binary,
+  packaged with nfpm ([`packaging/nfpm.yaml`](packaging/nfpm.yaml)). No setup
+  (uses `GITHUB_TOKEN`).
+- **Homebrew** (`rayforcedb/tap`) — a **build-from-source** formula
+  ([`packaging/homebrew-formula.rb.tmpl`](packaging/homebrew-formula.rb.tmpl)),
+  auto-bumped by the `homebrew` job. Compiling on the user's machine means all
+  Mac arches work and there's no redistribution-portability footgun.
+
+> Want maximum per-CPU performance instead of a portable binary? Build from
+> source — `make` and Homebrew both default to `-march=native`. Only the
+> *distributed* x86-64 artifacts use the `x86-64-v3` baseline.
 
 ## Platform support
 

docs/javascripts/main.js

@@ -180,7 +180,7 @@ function rfUpdateBannerOffset() {
 
   const REPO = 'RayforceDB/rayforce';
   const CACHE_KEY = 'gh-release:' + REPO;
-  const CACHE_TTL = 60 * 60 * 1000; /* 1 hour */
+  const REVALIDATE_AFTER = 5 * 60 * 1000; /* refetch the latest release at most this often */
 
   /* Only Linux x86_64 and macOS (arm64) ship today; Windows/mobile/unknown fall
    * through to the releases page. navigator.platform can't tell Apple Silicon
@@ -236,34 +236,46 @@ function rfUpdateBannerOffset() {
   function readCache() {
     try {
       const v = JSON.parse(localStorage.getItem(CACHE_KEY));
-      if (!v || (Date.now() - v.t) > CACHE_TTL) return null;
-      return v.rel;
+      return (v && v.rel && v.rel.tag) ? v : null;
     } catch (e) { return null; }
   }
   function writeCache(rel) {
     try { localStorage.setItem(CACHE_KEY, JSON.stringify({ t: Date.now(), rel: rel })); } catch (e) {}
   }
+  /* Persist always, but repaint only when the latest tag actually changed — so
+     a fresh release shows up without re-flipping the banner on every load. */
+  function update(rel) {
+    const changed = !cached || cached.rel.tag !== rel.tag;
+    writeCache(rel);
+    if (changed) apply(rel);
+  }
 
+  /* Stale-while-revalidate: paint the cached release instantly (may be slightly
+     stale), then revalidate in the background and repaint only if it changed.
+     Without this, a cached value froze the banner on the old version until its
+     TTL — a new release wouldn't show for up to an hour. */
   const cached = readCache();
-  if (cached) { apply(cached); return; }
+  if (cached) apply(cached.rel);
+  if (cached && (Date.now() - cached.t) < REVALIDATE_AFTER) return; /* fresh enough; skip the network */
 
-  /* Primary: ungh.cc. Shape: { release: { tag, assets: [{ downloadUrl }] } }. */
-  fetch('https://ungh.cc/repos/' + REPO + '/releases/latest')
-    .then(r => r.ok ? r.json() : Promise.reject(new Error('ungh ' + r.status)))
+  /* GitHub's API is authoritative and fresh; ungh.cc (CDN-cached, can lag
+     GitHub by ~a day) is the fallback when the direct API is unreachable. */
+  fetch('https://api.github.com/repos/' + REPO + '/releases/latest', { headers: { Accept: 'application/vnd.github+json' } })
+    .then(r => r.ok ? r.json() : Promise.reject(new Error('gh ' + r.status)))
     .then(d => {
-      const r = d.release || d;
-      const rel = { tag: r.tag, urls: (r.assets || []).map(a => a.downloadUrl).filter(Boolean) };
-      if (!rel.tag) throw new Error('ungh shape');
-      writeCache(rel); apply(rel);
+      const rel = { tag: d.tag_name, urls: (d.assets || []).map(a => a.browser_download_url).filter(Boolean) };
+      if (!rel.tag) throw new Error('gh shape');
+      update(rel);
     })
     .catch(() => {
-      /* Fallback: GitHub API. Shape: { tag_name, assets: [{ browser_download_url }] }. */
-      return fetch('https://api.github.com/repos/' + REPO + '/releases/latest', { headers: { Accept: 'application/vnd.github+json' } })
-        .then(r => r.ok ? r.json() : Promise.reject(new Error('gh ' + r.status)))
+      /* Fallback: ungh.cc. Shape: { release: { tag, assets: [{ downloadUrl }] } }. */
+      return fetch('https://ungh.cc/repos/' + REPO + '/releases/latest')
+        .then(r => r.ok ? r.json() : Promise.reject(new Error('ungh ' + r.status)))
         .then(d => {
-          const rel = { tag: d.tag_name, urls: (d.assets || []).map(a => a.browser_download_url).filter(Boolean) };
-          if (!rel.tag) throw new Error('gh shape');
-          writeCache(rel); apply(rel);
+          const r = d.release || d;
+          const rel = { tag: r.tag, urls: (r.assets || []).map(a => a.downloadUrl).filter(Boolean) };
+          if (!rel.tag) throw new Error('ungh shape');
+          update(rel);
         });
     })
     .catch(() => { /* CTAs stay on /releases/latest; banner keeps its default text. */ });

packaging/homebrew-formula.rb.tmpl

@@ -0,0 +1,30 @@
+# Homebrew formula TEMPLATE for the rayforcedb/tap tap.
+#
+# release.yml substitutes __URL__ and __SHA256__ (the GitHub source tarball for
+# the tag and its sha256) and pushes the result to RayforceDB/homebrew-tap as
+# Formula/rayforce.rb on every release.
+#
+# Build-from-source on purpose: it compiles for the user's own CPU (so it works
+# on every Mac arch and never SIGILLs the way a redistributed -march=native
+# binary can), and a zero-dependency `make` build needs no `depends_on`.
+class Rayforce < Formula
+  desc "Embeddable columnar analytics and graph traversal engine in pure C"
+  homepage "https://rayforcedb.com/"
+  url "__URL__"
+  sha256 "__SHA256__"
+  license "MIT"
+  head "https://github.com/RayforceDB/rayforce.git", branch: "master"
+
+  def install
+    # Inject the release version (no .git in a source tarball, so git-describe
+    # can't), build optimized for the user's machine, install binary + header.
+    system "make", "release", "RAY_VERSION=#{version}"
+    bin.install "rayforce"
+    include.install "include/rayforce.h"
+  end
+
+  test do
+    assert_match "usage", shell_output("#{bin}/rayforce --help")
+    assert_match version.to_s, pipe_output("#{bin}/rayforce", "(.sys.build)\n")
+  end
+end

packaging/nfpm.yaml

@@ -0,0 +1,32 @@
+# nfpm config — produces the Rayforce .deb.
+#
+# Driven by .github/workflows/release.yml on the Linux build, which first
+# rebuilds a PORTABLE binary (`make release RAY_MARCH=x86-64-v3`, NOT the
+# default -march=native) so the package runs on any modern x86-64 CPU, then
+# runs `nfpm pkg --packager deb`.  $RAY_VERSION is expanded from the env.
+#
+# To also emit .rpm / .apk later, just run nfpm with --packager rpm / apk
+# (same config) — no changes needed here.
+name: "rayforce"
+arch: "amd64"
+platform: "linux"
+version: "${RAY_VERSION}"
+maintainer: "Anton Kundenko <anton.kundenko@gmail.com>"
+description: |
+  Embeddable columnar analytics and graph traversal engine in pure C.
+  Built-in Rayfall query language, IPC, datalog, and vector search. Zero deps.
+vendor: "RayforceDB"
+homepage: "https://rayforcedb.com/"
+license: "MIT"
+section: "database"
+priority: "optional"
+
+contents:
+  - src: ./rayforce
+    dst: /usr/bin/rayforce
+  - src: ./include/rayforce.h
+    dst: /usr/include/rayforce.h
+  - src: ./LICENSE
+    dst: /usr/share/doc/rayforce/copyright
+  - src: ./README.md
+    dst: /usr/share/doc/rayforce/README.md

src/lang/eval.c

@@ -449,7 +449,19 @@ ray_t* atomic_map_binary_op(ray_binary_fn fn, uint16_t dag_opcode, ray_t* left,
 
     int64_t len;
     if (left_coll && right_coll) {
-        len = ray_len(left) < ray_len(right) ? ray_len(left) : ray_len(right);
+        /* Both operands are vectors: lengths MUST match.  Silently
+         * truncating to the shorter (the prior behaviour) hid a real
+         * error and diverged from the compiled/VM path, which already
+         * rejects the mismatch (src/ops/expr.c).  A 1-element vector
+         * does NOT broadcast (only a true atom does — see the
+         * !left_coll/!right_coll branch). */
+        if (ray_len(left) != ray_len(right)) {
+            const char* opn = ray_opcode_name(dag_opcode);
+            return ray_error("length", "%s: operand lengths must match, got %lld and %lld",
+                             opn ? opn : "binary op",
+                             (long long)ray_len(left), (long long)ray_len(right));
+        }
+        len = ray_len(left);
     } else {
         len = left_coll ? ray_len(left) : ray_len(right);
     }
@@ -2964,6 +2976,7 @@ static void ray_register_builtins(void) {
     register_unary("dl-eval",      RAY_FN_NONE, ray_dl_eval_fn);
     register_binary("dl-query",    RAY_FN_NONE, ray_dl_query_fn);
     register_binary("dl-provenance", RAY_FN_NONE, ray_dl_provenance_fn);
+    register_unary("dl-free",      RAY_FN_NONE, ray_dl_free_fn);
 
     /* Vector similarity / embeddings / HNSW */
     register_binary("cos-dist",    RAY_FN_NONE, ray_cos_dist_fn);

src/lang/internal.h

@@ -516,6 +516,7 @@ ray_t* ray_dl_stratify_fn(ray_t* x);
 ray_t* ray_dl_eval_fn(ray_t* x);
 ray_t* ray_dl_query_fn(ray_t* prog_obj, ray_t* pred_obj);
 ray_t* ray_dl_provenance_fn(ray_t* prog_obj, ray_t* pred_obj);
+ray_t* ray_dl_free_fn(ray_t* x);
 void   ray_dl_reset_rules(void);
 
 /* System builtins (formerly static in eval.c, now in system.c) */