ci: harden workflow token handling #569

Workflow file for this run

	name: CI
	permissions:
	contents: read

	on:
	push:
	branches: [main, master]
	pull_request:
	branches: [main, master]
	# Cancel in-flight runs of the same workflow when a new commit is pushed.
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.sha }}
	cancel-in-progress: true
	env:
	CARGO_TERM_COLOR: always
	CARGO_INCREMENTAL: "0" # CI builds don't benefit from incremental; smaller cache.
	CARGO_PROFILE_DEV_DEBUG: "0" # Reduces target/ size, improves cache hit rate.
	RUSTFLAGS: -Dwarnings
	# Testing strategy grounded in real-world ANN library issues:
	# - hnswlib #635: M vs Mcurmax bug in neighbor selection
	# - hnswlib #626: Use-after-free in deletion
	# - hnswlib #592: Vector not normalized for cosine distance
	# - hnswlib #608: Issues after deleting vectors
	# - faiss #4295: Integer overflow on large datasets
	# - usearch #405: Quantization issues with i8 + inner product
	# See docs/TESTING.md for full details
	jobs:
	# Cheap fan-in gate: fmt + clippy + basic check on ubuntu.
	# Failure here aborts every downstream matrix job (saves ~5-10 minutes per
	# PR on a fmt typo). Pattern from tokio-rs/tokio.
	basics:
	name: Basics (fmt + clippy + check)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	components: rustfmt, clippy
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- name: Check formatting
	run: cargo fmt --check
	- name: Clippy (hnsw)
	run: cargo clippy --no-default-features --features hnsw --all-targets -- -D warnings
	- name: Clippy (all features)
	run: cargo clippy --all-features --all-targets -- -D warnings
	# Primary test suite (x86) -- nextest for parallel test-binary execution.
	test:
	name: Test (ubuntu)
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
	with:
	tool: cargo-nextest
	- name: Build
	run: cargo build --no-default-features --features hnsw
	- name: Test (unit + integration)
	run: cargo nextest run --no-default-features --features hnsw
	- name: Test (sq4)
	run: cargo nextest run --no-default-features --features hnsw,sq4
	- name: Test (sq8)
	run: cargo nextest run --no-default-features --features hnsw,sq8
	- name: Test (lemur)
	run: cargo nextest run --no-default-features --features lemur
	- name: Test (all features, lib only)
	run: cargo nextest run --lib --all-features
	env:
	RUSTFLAGS: ""
	- name: Test (all-feature integration coverage)
	run: >
	cargo nextest run --all-features
	--test prop_ann_universal
	--test cross_algorithm_consistency
	--test correctness_regression
	--test recall_ground_truth
	--test diskann_persistence_test
	env:
	RUSTFLAGS: ""
	# nextest does not support doctests yet (nextest-rs/nextest#16).
	- name: Doc tests
	run: cargo test --doc --no-default-features --features hnsw
	- name: Run example assertions
	run: \|
	for ex in 03_quick_benchmark sift_benchmark lid_demo; do
	cargo run --no-default-features --features hnsw --release --example "$ex"
	done
	cargo run --no-default-features --features hnsw --release --example acorn_selectivity -- \
	--n 500 --queries 50 --k 10 --neighbors 16 --json
	cargo run --no-default-features --features hnsw,filtered_graph,range_filtered,curator --release --example acorn_selectivity -- \
	--n 200 --queries 10 --k 5 --neighbors 12 --json
	- name: ann_benchmark smoke
	run: \|
	python scripts/generate_ann_smoke_data.py /tmp/vicinity-ann-ci --train 64 --test 12 --dim 8 --k 10
	python scripts/generate_sparse_mips_smoke_data.py /tmp/vicinity-sparse-ci --train 64 --test 12 --vocab 256 --topics 8 --nnz 16 --k 10
	cargo test --example ann_benchmark --no-default-features --features hnsw,serde
	cargo run --no-default-features --features sparse_mips --release --example sparse_mips_benchmark -- \
	/tmp/vicinity-sparse-ci
	cargo run --no-default-features --features hnsw,serde,nsw,lsh,kdtree,balltree,rptree,kmeans_tree,vamana,diskann,filtered_graph,range_filtered,curator,fresh_graph,store --release --example ann_benchmark -- \
	/tmp/vicinity-ann-ci \
	--algo hnsw --algo nsw --algo lsh \
	--algo inplace --algo vamana --algo diskann --algo store \
	--algo filtered_graph --algo curator --algo range_filtered \
	--algo fresh_graph --algo fresh_graph_churn --algo inplace_churn --algo lsm_churn \
	--algo kdtree --algo balltree --algo rptree --algo rp_forest --algo kmeans_tree \
	--ef-search 10 \
	--tree-leaf-sizes 4 --tree-depths 8 --rp-num-trees 3 \
	--kmeans-clusters 4 --kmeans-leaf-sizes 8 --kmeans-depths 4 --kmeans-iters 3 \
	--churn-base-size 32 --churn-cycles 8 --churn-queries 6 \
	--snapshot-load --json --fresh --results /tmp/vicinity-ann-ci-graph-classic.jsonl
	cargo run --no-default-features --features hnsw,ivf_pq,ivf_avq,ivf_rabitq,rp_quant,binary_index,sq4,sq8,lsh,persistence --release --example ann_benchmark -- \
	/tmp/vicinity-ann-ci \
	--algo ivfpq --algo ivf_avq --algo ivf_rabitq \
	--algo rp_quant --algo binary_index --algo sq4 --algo sq4u --algo sq8u --algo lsh \
	--ef-search 10 \
	--pq-clusters 8 --pq-codebooks 2 --pq-codebook-size 8 \
	--pq-training-sample-size 64 --pq-kmeans-max-iter 3 --pq-nprobes 1 \
	--snapshot-load --json --fresh --results /tmp/vicinity-ann-ci-quant.jsonl
	- name: CLI smoke
	run: \|
	printf '{"id":0,"vec":[1.0,0.0,0.0]}\n{"id":1,"vec":[0.0,1.0,0.0]}\n{"id":2,"vec":[0.7,0.7,0.0]}\n' > /tmp/v.jsonl
	cargo run --features cli --bin vicinity -- build /tmp/v.jsonl -o /tmp/vi.json
	out=$(cargo run --features cli --bin vicinity -- search /tmp/vi.json --query '[0.8,0.6,0.0]' -k 2)
	echo "$out"
	echo "$out" \| head -1 \| grep -q '^2' \|\| { echo "FAIL: doc 2 should rank first"; exit 1; }
	- name: Property tests
	run: cargo nextest run --no-default-features --features hnsw -E 'test(property_)'
	# Semver-checks only on PRs.
	semver:
	name: Semver check
	runs-on: ubuntu-latest
	needs: basics
	if: github.event_name == 'pull_request'
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
	with:
	tool: cargo-semver-checks
	- run: cargo semver-checks --default-features
	# ARM native testing (M1/M2 - tests NEON code paths)
	test-arm:
	name: Test (macos / ARM)
	runs-on: macos-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
	with:
	tool: cargo-nextest
	- name: Build (ARM)
	run: cargo build --no-default-features --features hnsw
	- name: Test (ARM)
	run: cargo nextest run --no-default-features --features hnsw
	# MSRV check
	msrv:
	name: MSRV (1.89)
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "1.89.0"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- name: Check MSRV
	run: \|
	rm -f Cargo.lock # v4 lock requires Cargo 1.83+
	cargo check --no-default-features --features hnsw
	# Feature-matrix compilation (catches cfg + optional-dep drift)
	feature-matrix:
	name: Feature matrix
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
	with:
	tool: cargo-hack
	- name: Compile each feature (no dev-deps)
	run: cargo hack check --each-feature --no-dev-deps --exclude-features persistence
	# Cross-compilation check for different targets
	cross-compile:
	name: Cross-compile (${{ matrix.target }})
	runs-on: ubuntu-latest
	needs: basics
	strategy:
	matrix:
	target:
	- x86_64-unknown-linux-gnu
	- aarch64-unknown-linux-gnu
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	targets: ${{ matrix.target }}
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- name: Check ${{ matrix.target }}
	run: cargo check --target ${{ matrix.target }} --no-default-features --features hnsw
	# Recall regression detection -- catches hnswlib #635-class issues.
	recall-regression:
	name: Recall regression
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- name: Build release
	run: cargo build --release --no-default-features --features hnsw
	- name: Run recall benchmark
	run: \|
	cargo run --release --example 02_measure_recall --no-default-features --features hnsw 2>&1 \| tee recall_output.txt
	# Fail if recall@10 drops below 80% at ef=100
	# This catches issues like hnswlib #635 (wrong M parameter)
	awk '/^[[:space:]]*100[[:space:]]/ {
	gsub(/%/, "", $2); found=1
	if ($2 + 0 < 80.0) { print "FAIL: ef=100 recall " $2 "% < 80%"; exit 1 }
	}
	END { if (!found) { print "FAIL: ef=100 row missing from output"; exit 1 } }' recall_output.txt
	# Regression tests for known bugs
	regression:
	name: Regression (known bugs)
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
	with:
	tool: cargo-nextest
	- name: Run regression tests
	run: cargo nextest run --test regression_known_bugs --no-default-features --features hnsw
	# Python bindings: build wheel, lint, type-check, stubtest, pytest.
	# Stubtest catches drift between the hand-written `_core.pyi` and the
	# compiled module the moment a Rust signature changes (voyager pattern).
	python:
	name: Python (pyvicinity / ${{ matrix.python-version }})
	runs-on: ubuntu-latest
	needs: basics
	strategy:
	matrix:
	python-version: ["3.10", "3.13"]
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: ${{ matrix.python-version }}
	# GH Actions runners don't have a venv, and maturin develop needs
	# one. Pin the venv via $VIRTUAL_ENV + $GITHUB_PATH so subsequent
	# steps inherit it. maturin develop is preferred over `maturin build
	# + pip install` because it drops `_core.so` directly into the source
	# tree, which avoids the cwd-shadows-site-packages problem when
	# stubtest imports `pyvicinity._core`.
	- name: Set up virtualenv
	run: \|
	python -m venv .venv
	echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
	echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> "$GITHUB_ENV"
	- name: Install build + test deps
	run: python -m pip install --upgrade pip maturin pytest mypy ruff numpy
	- name: Build extension (editable)
	run: maturin develop --release --features hnsw,python,parallel
	- name: Ruff
	run: python -m ruff check pyvicinity tests/test_python.py tests/test_download_ann_benchmarks.py tests/test_generate_sparse_mips_smoke_data.py tests/test_summarize_ann_results.py examples/python scripts/generate_ann_smoke_data.py scripts/generate_sparse_mips_smoke_data.py scripts/download_ann_benchmarks.py scripts/summarize_ann_results.py
	- name: Ruff format
	run: python -m ruff format --check pyvicinity tests/test_python.py tests/test_download_ann_benchmarks.py tests/test_generate_sparse_mips_smoke_data.py tests/test_summarize_ann_results.py examples/python scripts/generate_ann_smoke_data.py scripts/generate_sparse_mips_smoke_data.py scripts/download_ann_benchmarks.py scripts/summarize_ann_results.py
	- name: mypy --strict
	run: python -m mypy --strict pyvicinity
	- name: Stubtest (.pyi vs compiled module)
	run: python -m mypy.stubtest pyvicinity._core
	- name: Pytest
	run: python -m pytest tests/test_python.py tests/test_download_ann_benchmarks.py tests/test_generate_sparse_mips_smoke_data.py tests/test_summarize_ann_results.py -v
	- name: Python harness example
	run: python examples/python/03_ann_benchmarks_harness.py
	# Documentation build
	docs:
	name: Docs
	runs-on: ubuntu-latest
	needs: basics
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
	with:
	toolchain: "stable"
	- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version: "3.10"
	- name: Build docs
	run: cargo doc --all-features --no-deps
	env:
	RUSTDOCFLAGS: -Dwarnings

Navigation Menu

ci: harden workflow token handling #569

Workflow file

ci: harden workflow token handling #569

Uh oh!

Workflow file for this run