Skip to content
Rust

test(blob): cover mixed-nullability batches in one write #23017

Sign in to view logs

test(blob): cover mixed-nullability batches in one write

test(blob): cover mixed-nullability batches in one write #23017

Workflow file for this run

.github/workflows/rust.yml at c4b4515
name: Rust
on:
push:
branches:
- main
- release/**
pull_request:
branches:
- main
- release/**
paths:
- rust/**
- protos/**
- .github/workflows/rust.yml
- rust-toolchain.toml
- Cargo.toml
- Cargo.lock
- deny.toml
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
# This env var is used by Swatinem/rust-cache@v2 for the cache
# key, so we set it to make sure it is always consistent.
CARGO_TERM_COLOR: always
RUST_BACKTRACE: "1"
jobs:
format:
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
with:
components: rustfmt
- name: Check formatting
run: cargo fmt -- --check
package:
name: Check crates are publishable
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
# Catches manifest problems that only surface at publish time (missing
# readme/license files, path dependencies without a version, excluded
# files a crate needs). `--no-verify` skips building each packaged crate,
# which the other jobs already cover. Keep the excludes in sync with
# .github/workflows/cargo-publish.yml.
- name: Package crates
run: |
cargo package --workspace --no-verify \
--exclude lance-arrow-stats \
--exclude lance-arrow-scalar \
--all-features
rustdoc:
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Check documentation
run: RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps
clippy:
permissions:
contents: read
checks: write
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Get features
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | sort | uniq | paste -s -d "," -`
echo "ALL_FEATURES=${ALL_FEATURES}" >> $GITHUB_ENV
- name: Clippy
run: cargo clippy --profile ci --locked --features ${{ env.ALL_FEATURES }} --all-targets -- -D warnings
cargo-deny:
name: Check Rust dependencies (cargo-deny)
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: EmbarkStudios/cargo-deny-action@3fd3802e88374d3fe9159b834c7714ec57d6c979 # v2
with:
log-level: warn
command: check
linux-build:
runs-on: "ubuntu-24.04-8x"
timeout-minutes: 75
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang++
# Treat warnings as errors to catch issues early
RUSTFLAGS: "-D warnings"
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Setup rust toolchain
run: |
# Temporary mitigation for https://github.com/rust-lang/rust/issues/159261.
rustup toolchain install nightly-2026-07-13
rustup default nightly-2026-07-13
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Start DynamodDB and S3
run: docker compose -f docker-compose.yml up -d --wait
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov
- name: Run tests
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
cargo +nightly-2026-07-13 llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
- name: Upload coverage to Codecov
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
codecov_yml_path: codecov.yml
files: coverage.codecov
flags: unittests
fail_ci_if_error: false
linux-arm:
runs-on: ubuntu-24.04-arm64-8x
timeout-minutes: 75
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Setup rust toolchain
run: |
rustup toolchain install stable
rustup default stable
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Install dependencies
run: |
sudo apt -y -qq update
sudo apt install -y protobuf-compiler libssl-dev pkg-config
- name: Build tests
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
cargo test --profile ci --locked --features ${ALL_FEATURES} --no-run
- name: Start DynamodDB and S3
run: docker compose -f docker-compose.yml up -d --wait
- name: Run tests
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
cargo test --profile ci --locked --features ${ALL_FEATURES}
query-integration-tests:
runs-on: ubuntu-24.04-4x
timeout-minutes: 75
env:
# We use opt-level 1 which makes some tests 5x faster to run.
RUSTFLAGS: "-C debuginfo=1 -C opt-level=1"
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Setup rust toolchain
run: |
rustup toolchain install stable
rustup default stable
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
with:
cache-targets: false
cache-workspace-crates: true
- name: Install dependencies
run: |
sudo apt -y -qq update
sudo apt install -y protobuf-compiler libssl-dev pkg-config
- name: Build query integration tests
run: |
cargo build --locked -p lance --no-default-features --features fp16kernels,slow_tests --tests --test integration_tests
- name: Run query integration tests
run: |
cargo test --locked -p lance --no-default-features --features fp16kernels,slow_tests --test integration_tests
build-no-lock:
runs-on: ubuntu-24.04-8x
timeout-minutes: 30
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang++
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Set up Rust
run: |
rustup update stable
rustup default stable
# Remote cargo.lock to force a fresh build
- name: Remove Cargo.lock
run: rm -f Cargo.lock
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build all
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
cargo build --profile ci --benches --features ${ALL_FEATURES} --tests
mac-build:
runs-on: warp-macos-14-arm64-6x
timeout-minutes: 45
strategy:
matrix:
toolchain:
- stable
defaults:
run:
working-directory: ./rust
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Select new xcode
# Default XCode right now is 15.0.1, which contains a bug that causes
# backtraces to not show properly. See:
# https://github.com/rust-lang/rust/issues/113783
run: sudo xcode-select -s /Applications/Xcode_15.4.app
- name: Install dependencies
run: brew install protobuf
- name: Set up Rust
run: |
rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Build tests
run: |
cargo test --profile ci --locked --features fp16kernels,cli,dynamodb,substrait --no-run
- name: Run tests
run: |
cargo test --profile ci --features fp16kernels,cli,dynamodb,substrait
- name: Check benchmarks
run: |
cargo check --profile ci --benches --features fp16kernels,cli,dynamodb,substrait
windows-build:
runs-on: windows-latest-4x
defaults:
run:
working-directory: rust
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Install Protoc v21.12
working-directory: C:\
run: |
New-Item -Path 'C:\protoc' -ItemType Directory
Set-Location C:\protoc
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
7z x protoc.zip
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
shell: powershell
- name: Build tests
run: cargo test --profile ci --locked --no-run
- name: Run tests
run: cargo test --profile ci
- name: Check benchmarks
run: cargo check --profile ci --benches
qemu-pre-haswell:
# Verifies that lance-linalg's runtime SIMD dispatch still works
# correctly when the binary is built with the lower x86-64-v2 baseline
# (the legacy build path documented in CONTRIBUTING.md). Emulates a
# Nehalem CPU under qemu-user; catches any accidental AVX2/FMA
# instructions that leak past the runtime dispatch.
#
# The published-wheel default baseline (`target-cpu=haswell`) is set in
# `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to
# exercise the legacy path without affecting any other build.
name: pre-Haswell SIGILL check (qemu Nehalem)
runs-on: ubuntu-24.04
timeout-minutes: 60
env:
CC: clang
CXX: clang++
RUSTFLAGS: "-C target-cpu=x86-64-v2"
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem"
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Setup rust toolchain
run: |
rustup toolchain install stable
rustup default stable
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Restore QEMU 8.2.10
id: qemu-cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ runner.temp }}/lance-qemu/8.2.10/qemu-x86_64
key: qemu-user-8.2.10-x86_64-linux-user-${{ runner.os }}-${{ runner.arch }}-v1
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Install QEMU build dependencies
if: steps.qemu-cache.outputs.cache-hit != 'true'
run: |
sudo apt install -y \
ninja-build \
pkg-config \
libglib2.0-dev \
python3-venv \
curl \
xz-utils
- name: Build QEMU 8.2.10
if: steps.qemu-cache.outputs.cache-hit != 'true'
run: |
QEMU_VERSION=8.2.10
QEMU_ARCHIVE="qemu-${QEMU_VERSION}.tar.xz"
QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/${QEMU_VERSION}"
QEMU_SOURCE_DIR="${RUNNER_TEMP}/qemu-${QEMU_VERSION}"
# Ubuntu 24.04 ships QEMU 8.2.2, which is affected by
# https://gitlab.com/qemu-project/qemu/-/issues/2170.
curl --fail --location --silent --show-error \
"https://download.qemu.org/${QEMU_ARCHIVE}" \
--output "${RUNNER_TEMP}/${QEMU_ARCHIVE}"
echo "37b4a643da8ed6015eef35f5d7f06e7259d9c95359965a0a98e9667c621ab2bb ${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
| sha256sum --check -
tar --extract \
--file "${RUNNER_TEMP}/${QEMU_ARCHIVE}" \
--directory "${RUNNER_TEMP}"
mkdir "${QEMU_SOURCE_DIR}/build"
(
cd "${QEMU_SOURCE_DIR}/build"
../configure --target-list=x86_64-linux-user --disable-docs
ninja qemu-x86_64
)
mkdir -p "${QEMU_BIN_DIR}"
install -m 0755 \
"${QEMU_SOURCE_DIR}/build/qemu-x86_64" \
"${QEMU_BIN_DIR}/qemu-x86_64"
- name: Add QEMU to PATH
run: |
QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/8.2.10"
echo "${QEMU_BIN_DIR}" >> "${GITHUB_PATH}"
QEMU_VERSION_OUTPUT="$("${QEMU_BIN_DIR}/qemu-x86_64" --version)"
echo "${QEMU_VERSION_OUTPUT}"
if [[ "${QEMU_VERSION_OUTPUT}" != *"version 8.2.10"* ]]; then
echo "Expected QEMU 8.2.10, got: ${QEMU_VERSION_OUTPUT}" >&2
exit 1
fi
- name: Build lance-linalg lib tests in release mode
run: |
cargo test --release -p lance-linalg --lib --no-run
- name: Run lance-linalg lib tests under qemu Nehalem
run: |
cargo test --release -p lance-linalg --lib
msrv:
# Check the minimum supported Rust version
name: MSRV Check - Rust v${{ matrix.msrv }}
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang
CXX: clang++
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
submodules: true
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Install ${{ matrix.msrv }}
run: |
rustup toolchain install ${{ matrix.msrv }}
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
run: |
ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
cargo check --profile ci --workspace --tests --benches --features ${ALL_FEATURES}