docs(readme): plainer, generic README + repo description #100

Workflow file for this run

	name: CI

	# Runs the complete test methodology on every commit (any branch) and every pull request:
	# formatting + lint, unit/integration tests (gnu + musl, default and all-features), doc tests,
	# the boto3 S3 conformance suite, the crash-consistency (durability) harness, a two-node
	# replication soak, the warp-based stress harness, fuzz smoke tests, a benchmark compile check,
	# and coverage. Release artifacts are built by release.yml, not here.
	#
	# Build-once: the `build` job compiles the `cairn` binary a single time and uploads it as a
	# temporary artifact (retention-days: 1); every conformance job that just drives a prebuilt binary
	# `needs: build` and downloads it instead of recompiling — so the binary is compiled once, not ~15
	# times. The compile jobs that genuinely need their own build (lint/test/coverage, the `--features
	# fast-io` path, and the `--features failpoints` crash harnesses) keep building, but are now warmed by
	# a shared cargo cache (Swatinem/rust-cache, wired into ./.github/actions/setup).
	on:
	push:
	branches: ["**"]
	pull_request:

	concurrency:
	group: ci-${{ github.ref }}
	cancel-in-progress: true

	env:
	CARGO_TERM_COLOR: always
	RUST_BACKTRACE: 1
	# Harden cargo against flaky crates.io fetches on the runners (the "OpenSSL SSL_read:
	# unexpected eof" failures): retry transient downloads, and disable HTTP/2 multiplexing,
	# which is the known trigger for that early-EOF error.
	CARGO_NET_RETRY: "10"
	CARGO_HTTP_MULTIPLEXING: "false"

	# Least-privilege default for every job: CI only reads the repository (checkout) and writes nothing
	# back. Individual jobs override this if they ever need more. Closes the CodeQL
	# `actions/missing-workflow-permissions` alerts across all jobs.
	permissions:
	contents: read

	jobs:
	# Compile the plain debug `cairn` binary ONCE and publish it as a temporary artifact. Every
	# conformance job below that only drives a prebuilt binary downloads this instead of recompiling.
	build:
	name: build cairn (shared binary)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup # builds ui/dist (embedded into the binary) + toolchain + cache
	- run: cargo build --bin cairn
	# Conformance asserts behaviour, not Rust-level state, so it does not need debug symbols.
	# Stripping shrinks the artifact from ~260 MB to ~25 MB — far faster to upload once + download many.
	- run: strip target/debug/cairn
	- uses: actions/upload-artifact@v7
	with:
	name: cairn-bin
	path: target/debug/cairn
	# The `cleanup` job (end of file) deletes this the moment the run finishes; retention is a
	# fallback cap that bounds it to 1 day only if that job is somehow skipped.
	retention-days: 1
	if-no-files-found: error

	lint:
	name: fmt + clippy
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup
	with:
	components: rustfmt, clippy
	- run: cargo fmt --all --check
	- run: cargo clippy --workspace --all-targets -- -D warnings
	- run: cargo clippy --workspace --all-targets --all-features -- -D warnings

	test:
	name: test (${{ matrix.target }})
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	target:
	- x86_64-unknown-linux-gnu
	- x86_64-unknown-linux-musl
	steps:
	- uses: actions/checkout@v7
	- name: Install musl tools
	if: endsWith(matrix.target, 'musl')
	run: sudo apt-get update && sudo apt-get install -y musl-tools
	- uses: ./.github/actions/setup
	with:
	targets: ${{ matrix.target }}
	- uses: taiki-e/install-action@nextest
	# The libSQL/Turso backends (cairn-meta-async) bundle C deps that are not static-musl-safe
	# (they SIGSEGV under a fully static musl link). They are optional, glibc-only backends and are
	# NOT part of the default static musl binary (which links only rusqlite), so exclude them from
	# the musl run; the gnu and all-features jobs exercise them on glibc.
	- name: Test
	run: \|
	EXTRA=""
	case "${{ matrix.target }}" in
	*musl) EXTRA="--exclude cairn-meta-async" ;;
	esac
	cargo nextest run --workspace $EXTRA --target ${{ matrix.target }}
	cargo test --workspace $EXTRA --doc

	test-all-features:
	name: test (all features)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- name: Free disk space
	uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: false
	large-packages: false
	- uses: ./.github/actions/setup
	- uses: taiki-e/install-action@nextest
	# Exercises the optional backends (libSQL + Turso) and the fast-io (io_uring + kTLS) paths.
	- run: cargo nextest run --workspace --all-features

	# ---- conformance jobs: download the shared binary, drive it, no recompile -------------------

	conformance:
	name: boto3 conformance
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Drive Cairn with the AWS SDK (real SigV4 + aws-chunked streaming, the F-5 path)
	run: BIN=target/debug/cairn PY=python3 bash conformance/run.sh

	checksums:
	name: modern-SDK flexible checksums
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	# botocore[crt] lets boto3 emit CRC32C + CRC64NVME so the harness exercises every algorithm.
	- run: pip install 'boto3' 'botocore[crt]'
	- name: Assert checksums are computed, stored, and echoed on PUT/GET/HEAD (ARCH 21.1)
	run: BIN=target/debug/cairn PY=python3 bash conformance/checksums.sh

	scrub:
	name: integrity scrub (bit-rot detection)
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Corrupt a stored blob on disk; assert the background scrub detects the ETag mismatch
	run: BIN=target/debug/cairn PY=python3 bash conformance/scrub.sh

	object-lock:
	name: object lock / WORM
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: COMPLIANCE/GOVERNANCE retention + legal hold enforced via the AWS SDK
	run: BIN=target/debug/cairn PY=python3 bash conformance/object_lock.sh

	notifications:
	name: webhook event notifications
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Event-notification delivery + HMAC signing (management UI listener on)
	run: BIN=target/debug/cairn PY=python3 bash conformance/notifications.sh

	sts:
	name: STS temporary credentials
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Mint + enforce scoped session credentials over a real SDK (management UI listener on)
	run: BIN=target/debug/cairn PY=python3 bash conformance/sts.sh

	console-session:
	name: console session-cookie auth
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	# Pure curl (no SDK): the httpOnly cookie flow is plain HTTP. Management UI listener on.
	- name: httpOnly session-cookie auth + S3 data-plane port isolation (management UI listener on)
	run: BIN=target/debug/cairn bash conformance/console_session.sh

	backup-restore:
	name: backup / restore / integrity e2e
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	# Pure curl (Bearer auth, no SDK): backup -> corrupt -> restore-to-fresh-dir -> integrity --repair.
	- name: Snapshot fidelity + reconcile baseline + surgical integrity --repair
	run: BIN=target/debug/cairn bash conformance/backup_restore.sh

	fast-io-conformance:
	name: fast-io sendfile conformance + keep-alive engagement
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	# The fast path is a build-time feature (off by default), so this job needs its own binary and
	# cannot reuse the shared artifact. Build the experimental zero-copy GET fast path and prove the
	# full S3 lifecycle is correct through it with the fast path maximally engaged (FASTIO_MIN_BYTES=0)...
	- run: cargo build --bin cairn --features fast-io
	- name: Full boto3 lifecycle through the sendfile fast path
	run: CAIRN_FASTIO_MIN_BYTES=0 BIN=target/debug/cairn PY=python3 bash conformance/run.sh
	# ...and that a pooled client engages the zero-copy path on EVERY request on a connection, not
	# just the first (the keep-alive rewrite's reason for being).
	- name: Sendfile keep-alive engagement
	run: BIN=target/debug/cairn bash conformance/sendfile_keepalive.sh

	# conformance/mesh.sh (5-node full-mesh replication) is intentionally NOT gated here: it needs the
	# dev sandbox disabled and stands up five node processes on dedicated ports — too heavy and
	# environment-specific for the shared runner. Run it manually: `bash conformance/mesh.sh`.

	rotation:
	name: master-key rotation e2e (#29)
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- name: Add a key, re-wrap stored secrets onto it, retire the old key (sharded); assert the fail-closed retire-gate
	run: BIN=target/debug/cairn PY=python3 bash conformance/rotation.sh

	share:
	name: object sharing e2e
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- name: Persistent share tokens + SigV4 presigned URLs, end to end
	run: BIN=target/debug/cairn bash conformance/share.sh

	concurrency:
	name: concurrency contention (single-key races)
	needs: build
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- name: Race N clients on one key (create / CAS / last-writer); assert atomic, no corruption
	run: BIN=target/debug/cairn PY=python3 N=32 bash conformance/concurrency.sh

	crash-consistency:
	name: crash consistency (F-4)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup
	# Self-builds `cairn --features failpoints` (the durability fault seams) — a different binary
	# from the shared artifact, so this job keeps its own compile (warmed by the shared cache).
	- name: Arm the durability fault seam, crash in the window, assert reconcile reclaims
	run: bash conformance/crash_consistency.sh

	crash-multipoint:
	name: durability multi-point crash
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	# Self-builds `cairn --features failpoints` (see crash_multipoint.sh) — keeps its own compile.
	- name: Crash at every blob-commit seam (PutObject + multipart); assert reconcile reclaims each orphan
	run: BIN=target/debug/cairn PY=python3 bash conformance/crash_multipoint.sh

	soak:
	name: replication soak (multi-host)
	needs: build
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Two-node replication soak — sustained PUTs, byte-identical verify, RSS leak check
	run: BIN=target/debug/cairn PY=python3 DURATION=90 bash conformance/soak.sh

	replication-chaos:
	name: replication chaos (fault injection)
	needs: build
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- run: pip install boto3
	- name: Break replication on purpose (target down, source SIGKILL, rapid overwrite); assert no data loss
	run: BIN=target/debug/cairn PY=python3 bash conformance/replication_chaos.sh

	stress:
	name: stress + stability (throughput, escalation, leak)
	needs: build
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	# warp put/get/mixed + a concurrency ramp + RSS/queue sampling, asserting zero op-errors, a live
	# server after the ramp, and no memory leak. The unified successor to warp.sh + warp_escalate.sh
	# (a strict superset). Runs the debug binary (fast); the stability assertions hold on debug —
	# absolute throughput is not gated, only correctness/liveness/leak. LEAK_PCT is relaxed for the
	# shared runner. `warp.sh` / `warp_escalate.sh` remain for focused/manual use.
	- name: Drive warp + concurrency ramp; assert zero errors, alive, no RSS leak
	run: BIN=target/debug/cairn DURATION=8s LEVELS="8 32 128 256" ESC_DURATION=6s LEAK_PCT=50 bash conformance/stress.sh

	blob-limits:
	name: blob store limits (out-of-space, huge, many)
	needs: build
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7
	- uses: actions/download-artifact@v8
	with:
	name: cairn-bin
	path: target/debug
	- run: chmod +x target/debug/cairn
	- uses: actions/setup-python@v6
	with:
	python-version: "3.x"
	- name: Out-of-space 507 on a constrained FS, huge-object round-trip, many objects paginated
	run: BIN=target/debug/cairn PY=python3 bash conformance/blob_limits.sh

	fuzz-smoke:
	name: fuzz smoke (decoder, xml, policy, compress, ids)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: dtolnay/rust-toolchain@nightly
	# Build cargo-fuzz with the default (stable) toolchain from rust-toolchain.toml — its deps
	# (rustix) don't compile on the very latest nightly. The fuzz RUN below then uses nightly via
	# RUSTUP_TOOLCHAIN (which overrides the stable toolchain file) for the -Z sanitizer build, and
	# that env propagates to cargo-fuzz's nested `cargo build`.
	- name: Install cargo-fuzz
	run: cargo install cargo-fuzz --locked
	# These fuzz targets live in crates that do not embed the UI, so no UI build is needed.
	- name: Fuzz smoke (nightly build)
	env:
	RUSTUP_TOOLCHAIN: nightly
	run: \|
	(cd crates/cairn-protocol && cargo fuzz run chunked_decoder -- -max_total_time=45)
	(cd crates/cairn-xml && cargo fuzz run request_parsers -- -max_total_time=30)
	(cd crates/cairn-authz && cargo fuzz run parse_policy -- -max_total_time=30)
	(cd crates/cairn-blob && cargo fuzz run compress_reader -- -max_total_time=45)
	(cd crates/cairn-types && cargo fuzz run parse_ids -- -max_total_time=30)

	benches:
	name: benchmarks compile
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/setup
	- run: cargo bench --workspace --no-run

	coverage:
	name: coverage
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- name: Free disk space
	uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: false
	large-packages: false
	- uses: ./.github/actions/setup
	with:
	components: llvm-tools-preview
	- uses: taiki-e/install-action@cargo-llvm-cov
	- run: cargo llvm-cov --workspace --lcov --output-path lcov.info
	- uses: actions/upload-artifact@v7
	with:
	name: coverage
	path: lcov.info

	# Delete the temporary shared binary once every job that consumes it has finished, so no artifact
	# lingers after the run. `if: always()` runs it even if a consumer failed (or the build failed and
	# the consumers were skipped — then there is nothing to delete and it no-ops). Uses the built-in
	# token via `gh api`, no third-party action. NOTE: keep `needs` in sync with the artifact consumers.
	cleanup:
	name: delete shared binary artifact
	needs:
	- conformance
	- checksums
	- scrub
	- object-lock
	- notifications
	- sts
	- console-session
	- backup-restore
	- rotation
	- share
	- concurrency
	- soak
	- replication-chaos
	- stress
	- blob-limits
	if: always()
	runs-on: ubuntu-latest
	permissions:
	actions: write # required to delete a run artifact
	steps:
	- name: Delete the cairn-bin artifact
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	set -euo pipefail
	id="$(gh api "/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts" \
	--jq '.artifacts[] \| select(.name=="cairn-bin") \| .id')"
	if [ -n "$id" ]; then
	gh api -X DELETE "/repos/${{ github.repository }}/actions/artifacts/$id"
	echo "deleted cairn-bin artifact (id=$id)"
	else
	echo "no cairn-bin artifact to delete (build skipped/failed, or already removed)"
	fi

Navigation Menu

docs(readme): plainer, generic README + repo description #100

Workflow file

docs(readme): plainer, generic README + repo description #100

Uh oh!

Workflow file for this run