From f4b16c2481ca961945339d0c079331e5d348c8d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 16 May 2026 15:40:53 +0000 Subject: [PATCH] feat(batman,bats-lane): default to tap-dancer format-ndjson --split Switch every test-output surface to amarbel-llc/tap's new split mode so stdout (or, for batsLane, the inline build-log echo) carries only failure NDJSON records, while passing records land in a side-channel file. This dramatically reduces the noise an interactive user or build log has to scroll through. - bats wrapper (nix/packages/batman.nix): pipe through `tap-dancer format-ndjson --split --pass-out` by default. New flags `--no-split` / `--full-output` restore the TAP-14 stream; `--pass-out PATH` overrides the per-run tmpdir for passing records. The hand-rolled hide-passing TAP filter is removed. - batman orchestrator (packages/batman/src/batman.ts): drop the duplicate TS-side hide-passing filter; pass `--no-split` through to the bats wrapper when `--full-output` / `--no-split` is set. - batsLane (nix/packages/bats-lane.nix): new `splitNdjson ? true` option layered on `emitNdjson`. Default produces `$out/run.failures.ndjson` + `$out/run.passes.ndjson`; the inline stderr echo carries failures only with a one-line pass-count summary. `splitNdjson = false` keeps the legacy combined `run.ndjson` and full echo. - Tests + docs updated: bats_wrapper.bats's four hide-passing tests replaced with split / no-split / pass-out / alias coverage; batman and bats-lane manpages updated. --- nix/packages/batman.nix | 65 ++++++++++---- nix/packages/bats-lane.nix | 79 +++++++++++++--- packages/batman/doc/bats-lane.7.scd | 34 ++++++- packages/batman/doc/bats-testing.7.scd | 19 +++- packages/batman/src/batman.ts | 89 +++---------------- packages/batman/zz-tests_bats/batman.bats | 2 +- .../batman/zz-tests_bats/bats_wrapper.bats | 89 ++++++++++--------- .../zz-tests_bats/ndjson_failure_demo.bats | 6 +- 8 files changed, 223 insertions(+), 160 deletions(-) diff --git a/nix/packages/batman.nix b/nix/packages/batman.nix index aa45966e70..b70034a5a1 100644 --- a/nix/packages/batman.nix +++ b/nix/packages/batman.nix @@ -163,7 +163,9 @@ let sandbox=true allow_local_binding=false no_tempdir_cleanup=false - hide_passing=false + split=true + pass_out="" + config="" bats_args=() while (( $# > 0 )); do @@ -180,8 +182,20 @@ let no_tempdir_cleanup=true shift ;; - --hide-passing) - hide_passing=true + --no-split|--full-output) + split=false + shift + ;; + --pass-out) + if (( $# < 2 )); then + echo "bats wrapper: --pass-out requires a path argument" >&2 + exit 2 + fi + pass_out="$2" + shift 2 + ;; + --pass-out=*) + pass_out="''${1#--pass-out=}" shift ;; --) @@ -197,6 +211,27 @@ let done set -- "''${bats_args[@]}" + # Per-run tmpdir for the passing-test NDJSON when split is on and + # no caller-supplied path was given. Reported on stderr so an + # interactive user can find it. Cleaned with the wrapper's EXIT + # trap unless --no-tempdir-cleanup is set. + pass_tmp="" + if $split && [[ -z "$pass_out" ]]; then + pass_tmp="$(mktemp -d --suffix=.batman)" + pass_out="$pass_tmp/passes.ndjson" + echo "bats wrapper: passing-test NDJSON -> $pass_out" >&2 + fi + + cleanup() { + if [[ -n "$config" ]]; then + rm -f "$config" + fi + if [[ -n "$pass_tmp" ]] && ! $no_tempdir_cleanup; then + rm -rf "$pass_tmp" + fi + } + trap cleanup EXIT + # Append batman's bats-libs to BATS_LIB_PATH (caller paths take precedence) export BATS_LIB_PATH="''${BATS_LIB_PATH:+$BATS_LIB_PATH:}${bats-libs}/share/bats" @@ -213,24 +248,17 @@ let use_tap14=true fi - filter_tap() { - if $hide_passing; then - awk ' - /^ ---$/ { in_yaml = 1; if (show) print; next } - /^ \.\.\.$/ { in_yaml = 0; if (show) print; next } - in_yaml { if (show) print; next } - /^ok / { show = ($0 ~ /# [Ss][Kk][Ii][Pp]/ || $0 ~ /# [Tt][Oo][Dd][Oo]/); if (show) print; next } - /^not ok / { show = 1; print; next } - { show = 1; print } - ' + reformat_tap() { + if $use_tap14; then + tap-dancer reformat else cat fi } - reformat_tap() { - if $use_tap14; then - tap-dancer reformat + split_or_passthrough() { + if $split; then + tap-dancer format-ndjson --split --pass-out "$pass_out" else cat fi @@ -238,7 +266,6 @@ let if $sandbox; then config="$(mktemp --suffix=.json)" - trap 'rm -f "$config"' EXIT # fence config: denyRead blocks credential dirs; allowWrite # restricts writes to /tmp; empty allowedDomains denies all @@ -290,12 +317,12 @@ let set -- --no-tempdir-cleanup "$@" fi - fence --settings "$config" -- bats "$@" | filter_tap | reformat_tap + fence --settings "$config" -- bats "$@" | reformat_tap | split_or_passthrough else if $no_tempdir_cleanup; then set -- --no-tempdir-cleanup "$@" fi - bats "$@" | filter_tap | reformat_tap + bats "$@" | reformat_tap | split_or_passthrough fi ''; }; diff --git a/nix/packages/bats-lane.nix b/nix/packages/bats-lane.nix index 97cc035434..71ec48ea14 100644 --- a/nix/packages/bats-lane.nix +++ b/nix/packages/bats-lane.nix @@ -145,10 +145,13 @@ let # Opt-in: capture the bats TAP-14 stream and convert it to NDJSON # alongside the run. When true, `$out` becomes a DIRECTORY containing - # `run.tap`, `run.ndjson`, and `exit_code` (the bats exit status as a - # one-line decimal). When false (default), `$out` remains a single - # stamp file as before. Existing consumers that `stat result` as a - # file are unaffected unless they opt in. + # `run.raw.tap`, `run.tap`, `exit_code`, and one of: + # - `run.failures.ndjson` + `run.passes.ndjson` when `splitNdjson` + # is true (the default), or + # - `run.ndjson` (combined records) when `splitNdjson` is false. + # When `emitNdjson` itself is false (the original default), `$out` + # remains a single stamp file as before. Existing consumers that + # `stat result` as a file are unaffected unless they opt in. # # Requires `tap-dancer-go` either as a top-level arg of this file # (the default plumbed by flake.nix) or as a per-call override @@ -158,6 +161,15 @@ let # (`docs/rfcs/0001-test-result-ndjson-schema.md`). emitNdjson ? false, + # When `emitNdjson` is true, split NDJSON records into + # `$out/run.failures.ndjson` (failures + bail-outs) and + # `$out/run.passes.ndjson` (passing records) via + # `tap-dancer format-ndjson --split`. Defaults to true so build + # logs (and the inline stderr echo) aren't drowned in pass + # records. Set false to keep the combined `run.ndjson` and full + # inline echo of every record. + splitNdjson ? true, + # Per-call override of the `tap-dancer-go` derivation used when # `emitNdjson = true`. Falls back to the top-level `tap-dancer-go` # arg of this file. Ignored when `emitNdjson = false`. @@ -265,11 +277,22 @@ let ''; # NDJSON form: $out is a directory carrying `run.raw.tap`, - # `run.tap`, `run.ndjson`, and `exit_code`. The pipeline is: + # `run.tap`, NDJSON records (split or combined), and `exit_code`. + # The pipeline is: # # bats --formatter tap13 ... → run.raw.tap (TAP-13 + YAML) # tap-dancer reformat → run.tap (TAP-14 header prepended) - # tap-dancer format-ndjson → run.ndjson (per-test NDJSON records) + # tap-dancer format-ndjson → split or combined NDJSON + # + # When `splitNdjson` is true (the default), `format-ndjson --split` + # writes failure records to `run.failures.ndjson` (stdout) and + # passing records to `run.passes.ndjson` (--pass-out). The inline + # stderr echo then carries only the failure records, keeping the + # build log focused on what went wrong. + # + # When `splitNdjson` is false, the original combined behavior is + # preserved: every record lands in `run.ndjson` and is echoed + # inline. # # We disable errexit around the bats call so a failed test run # still gets converted to NDJSON before the derivation exits @@ -277,6 +300,37 @@ let # preserved via `nix build --keep-failed`. The bats exit status # is the gating signal; reformat/format-ndjson exit codes are # ignored so we don't double-fail or mask the bats outcome. + ndjsonRenderStep = + if splitNdjson then + '' + ${tapDancerGo}/bin/tap-dancer format-ndjson --split \ + --pass-out "$out/run.passes.ndjson" \ + < "$out/run.tap" > "$out/run.failures.ndjson" || true + # Ensure both files exist even on all-pass / all-fail runs, + # so downstream consumers can unconditionally read them. + [ -e "$out/run.failures.ndjson" ] || : > "$out/run.failures.ndjson" + [ -e "$out/run.passes.ndjson" ] || : > "$out/run.passes.ndjson" + '' + else + '' + ${tapDancerGo}/bin/tap-dancer format-ndjson \ + < "$out/run.tap" > "$out/run.ndjson" || true + ''; + ndjsonEchoStep = + if splitNdjson then + '' + printf '%s\n' '>>> BATSLANE NDJSON BEGIN <<<' >&2 + cat "$out/run.failures.ndjson" >&2 + pass_count=$(wc -l < "$out/run.passes.ndjson" | tr -d ' ') + printf 'passes: %s record(s) at %s\n' "$pass_count" "$out/run.passes.ndjson" >&2 + printf '%s\n' '>>> BATSLANE NDJSON END <<<' >&2 + '' + else + '' + printf '%s\n' '>>> BATSLANE NDJSON BEGIN <<<' >&2 + cat "$out/run.ndjson" >&2 + printf '%s\n' '>>> BATSLANE NDJSON END <<<' >&2 + ''; ndjsonInvocation = '' mkdir -p "$out" cd stage/zz-tests_bats @@ -292,18 +346,15 @@ let set -o errexit ${tapDancerGo}/bin/tap-dancer reformat \ < "$out/run.raw.tap" > "$out/run.tap" || cp "$out/run.raw.tap" "$out/run.tap" - ${tapDancerGo}/bin/tap-dancer format-ndjson \ - < "$out/run.tap" > "$out/run.ndjson" || true + ${ndjsonRenderStep} echo "$bats_status" > "$out/exit_code" - # Echo the NDJSON to stderr between sentinel markers so the - # nix builder log carries the captured records inline. On - # failure, `nix build` prints the build-log tail to the user + # Echo NDJSON to stderr between sentinel markers so the nix + # builder log carries the captured records inline. On failure, + # `nix build` prints the build-log tail to the user # automatically; for any run, `nix log ` retrieves the # full log including this block. Agents extracting the NDJSON # programmatically can sed/awk between the BEGIN/END markers. - printf '%s\n' '>>> BATSLANE NDJSON BEGIN <<<' >&2 - cat "$out/run.ndjson" >&2 - printf '%s\n' '>>> BATSLANE NDJSON END <<<' >&2 + ${ndjsonEchoStep} exit "$bats_status" ''; diff --git a/packages/batman/doc/bats-lane.7.scd b/packages/batman/doc/bats-lane.7.scd index b6e5a1b43b..5ff254dde8 100644 --- a/packages/batman/doc/bats-lane.7.scd +++ b/packages/batman/doc/bats-lane.7.scd @@ -192,15 +192,30 @@ testFiles = [ "current_version/*.bats" "previous_versions/main.bats" ]; The pipeline when emitNdjson is on: ``` -bats --formatter tap13 ... → run.raw.tap (TAP-13 + YAML diagnostics) -tap-dancer reformat → run.tap (TAP-14 with version header) -tap-dancer format-ndjson → run.ndjson (per-test NDJSON records) +bats --formatter tap13 ... → run.raw.tap (TAP-13 + YAML) +tap-dancer reformat → run.tap (TAP-14 + header) +tap-dancer format-ndjson [--split] → split or combined NDJSON ``` + With *splitNdjson = true* (the default), the final stage runs + *tap-dancer format-ndjson --split --pass-out run.passes.ndjson* + so failure records land in *run.failures.ndjson* and passing + records land in *run.passes.ndjson*. The inline build-log echo + then carries only the failure records. + + With *splitNdjson = false*, the original combined behavior is + preserved: every record lands in *run.ndjson*. + The TAP-13 formatter is selected automatically when no formatter is supplied via *extraBatsArgs*; callers passing their own *--formatter* / *--tap* are honored as-is. +*splitNdjson* + A bool (default true). Ignored unless *emitNdjson = true*. Set + false to opt out of the split-output default and produce a + single combined *run.ndjson* file plus a full inline build-log + echo. + *tapDancerGo* A derivation override (default: the top-level *tap-dancer-go* passed when this file is imported). Ignored unless @@ -213,7 +228,18 @@ When *emitNdjson = false* (the default), *$out* is a single empty regular file — a stamp signaling that bats succeeded. Consumers can treat *result* as a file and *stat* / hash it normally. -When *emitNdjson = true*, *$out* is a *directory* containing: +When *emitNdjson = true*, *$out* is a *directory*. Under the default +*splitNdjson = true*: + +``` +run.raw.tap # bats's raw TAP-13 output (pre-reformat) +run.tap # TAP-14 with version header (tap-dancer reformat) +run.failures.ndjson # NDJSON failure records (stdout of --split) +run.passes.ndjson # NDJSON passing records (--pass-out target) +exit_code # bats's exit status as a one-line decimal +``` + +Under *splitNdjson = false*: ``` run.raw.tap # bats's raw TAP-13 output (pre-reformat) diff --git a/packages/batman/doc/bats-testing.7.scd b/packages/batman/doc/bats-testing.7.scd index b2cd9f079e..388bc1b105 100644 --- a/packages/batman/doc/bats-testing.7.scd +++ b/packages/batman/doc/bats-testing.7.scd @@ -8,7 +8,10 @@ bats-testing - BATS integration test conventions for amarbel-llc projects Integration tests in amarbel-llc projects use BATS (Bash Automated Testing System) via the *batman* wrapper. Batman bundles BATS with six support -libraries, fence-based sandbox isolation, and TAP-14 output formatting. +libraries, fence-based sandbox isolation, and a split-NDJSON output +pipeline (failures on stdout, passes to a per-run tmpdir file) backed +by *tap-dancer format-ndjson --split*. Pass *--no-split* / *--full-output* +to restore the historical TAP-14 stream. This manpage covers the conventions for writing, organizing, and running BATS tests. @@ -288,8 +291,18 @@ arguments: *--no-tempdir-cleanup* Keep BATS temp directories after test run. -*--hide-passing* - Filter TAP output to show only failures and skips. +*--no-split*, *--full-output* + Bypass the default split-output pipeline and emit the bats + TAP-14 stream verbatim on stdout (plan, version, and every + *ok*/*not ok* line). Equivalent to the pre-split behavior. + +*--pass-out* _PATH_ + Override the path for the passing-test NDJSON file produced + by the default split pipeline. Implies the split mode is + active; the file is written when at least one test passes + and is otherwise an empty file. The wrapper still emits a + stderr diagnostic naming the chosen path. Ignored under + *--no-split*. *--diagnostics-stderr* Write batman's wrapper diagnostics (parse errors, missing diff --git a/packages/batman/src/batman.ts b/packages/batman/src/batman.ts index f34d1eb7d2..8ba1126b7f 100644 --- a/packages/batman/src/batman.ts +++ b/packages/batman/src/batman.ts @@ -21,7 +21,7 @@ import * as nodePath from "node:path"; type ParsedArgs = { noTempdirCleanup: boolean; - hidePassing: boolean; + fullOutput: boolean; dryRun: boolean; diagnosticsStderr: boolean; positional: string[]; @@ -35,7 +35,7 @@ function parseArgs(argv: string[]): ParsedArgs { const positional: string[] = []; let noTempdirCleanup = false; - let hidePassing = false; + let fullOutput = false; let dryRun = false; let diagnosticsStderr = false; @@ -45,8 +45,9 @@ function parseArgs(argv: string[]): ParsedArgs { case "--no-tempdir-cleanup": noTempdirCleanup = true; break; - case "--hide-passing": - hidePassing = true; + case "--no-split": + case "--full-output": + fullOutput = true; break; case "--dry-run": dryRun = true; @@ -62,14 +63,15 @@ function parseArgs(argv: string[]): ParsedArgs { } } - // --no-tempdir-cleanup is forwarded to bats too. - const passthrough = noTempdirCleanup - ? ["--no-tempdir-cleanup", ...passthroughBase] - : passthroughBase; + // Forward batman-level toggles that the bats wrapper also understands. + const forwarded: string[] = []; + if (noTempdirCleanup) forwarded.push("--no-tempdir-cleanup"); + if (fullOutput) forwarded.push("--no-split"); + const passthrough = [...forwarded, ...passthroughBase]; return { noTempdirCleanup, - hidePassing, + fullOutput, dryRun, diagnosticsStderr, positional, @@ -133,78 +135,22 @@ async function logDiagnostic( await fsp.appendFile(nodePath.join(dir, "batman.log"), `${ts} ${msg}\n`); } -// hide-passing TAP filter: strip passing `ok N ...` lines and their YAML blocks. -// Mirrors the awk used by the existing bats wrapper. -function makeHidePassingFilter(): (chunk: string) => string { - let buf = ""; - let inYaml = false; - let show = true; - return (chunk: string) => { - buf += chunk; - let out = ""; - let nl: number; - while ((nl = buf.indexOf("\n")) !== -1) { - const line = buf.slice(0, nl); - buf = buf.slice(nl + 1); - if (/^ ---$/.test(line)) { - inYaml = true; - if (show) out += line + "\n"; - continue; - } - if (/^ \.\.\.$/.test(line)) { - inYaml = false; - if (show) out += line + "\n"; - continue; - } - if (inYaml) { - if (show) out += line + "\n"; - continue; - } - if (/^ok /.test(line)) { - show = - / # [Ss][Kk][Ii][Pp]/.test(line) || / # [Tt][Oo][Dd][Oo]/.test(line); - if (show) out += line + "\n"; - continue; - } - if (/^not ok /.test(line)) { - show = true; - out += line + "\n"; - continue; - } - show = true; - out += line + "\n"; - } - return out; - }; -} - async function runGroup( dir: string, files: string[], passthrough: string[], - hidePassing: boolean, diagnosticsStderr: boolean, ): Promise { const cfg = nodePath.join(dir, "fence.jsonc"); const fileArgs = files.map((f) => nodePath.join(dir, f)); return new Promise((resolve) => { - const stdout = hidePassing ? "pipe" : "inherit"; const child = spawn( "fence", ["--settings", cfg, "--", "bats", ...passthrough, ...fileArgs], - { stdio: ["inherit", stdout, "inherit"] }, + { stdio: ["inherit", "inherit", "inherit"] }, ); - if (hidePassing && child.stdout) { - const filter = makeHidePassingFilter(); - child.stdout.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - const out = filter(chunk); - if (out.length > 0) process.stdout.write(out); - }); - } - child.on("error", (err) => { // Spawn failure (e.g. fence missing) - record and treat as group failure. void logDiagnostic(`spawn error in ${dir}: ${err.message}`, { @@ -240,8 +186,7 @@ async function main(): Promise { return 2; } - const { hidePassing, dryRun, diagnosticsStderr, positional, passthrough } = - parsed; + const { dryRun, diagnosticsStderr, positional, passthrough } = parsed; // Validate paths exist before discovery. for (const p of positional) { @@ -280,13 +225,7 @@ async function main(): Promise { let aggregate = 0; for (const [dir, files] of groups) { - const code = await runGroup( - dir, - files, - passthrough, - hidePassing, - diagnosticsStderr, - ); + const code = await runGroup(dir, files, passthrough, diagnosticsStderr); if (code !== 0) aggregate = 1; } return aggregate; diff --git a/packages/batman/zz-tests_bats/batman.bats b/packages/batman/zz-tests_bats/batman.bats index 6c7be8cab5..7ae4f00f59 100644 --- a/packages/batman/zz-tests_bats/batman.bats +++ b/packages/batman/zz-tests_bats/batman.bats @@ -12,7 +12,7 @@ teardown() { } function batman_splits_argv_at_double_dash { # @test - run "$BATMAN_BIN" --dry-run --hide-passing "$FIXTURES" -- --filter X + run "$BATMAN_BIN" --dry-run --full-output "$FIXTURES" -- --filter X assert_success # Dry-run output should reflect the positional ($FIXTURES) but not the # post-`--` flags. The post-`--` flags would never produce GROUP lines. diff --git a/packages/batman/zz-tests_bats/bats_wrapper.bats b/packages/batman/zz-tests_bats/bats_wrapper.bats index 35b72b8a57..fa7891e056 100644 --- a/packages/batman/zz-tests_bats/bats_wrapper.bats +++ b/packages/batman/zz-tests_bats/bats_wrapper.bats @@ -21,7 +21,7 @@ function truth { # @test true } EOF - run "$BATS_WRAPPER" --tap "${TEST_TMPDIR}/truth.bats" + run "$BATS_WRAPPER" --no-split --tap "${TEST_TMPDIR}/truth.bats" assert_success assert_output --partial "ok 1" } @@ -39,7 +39,7 @@ function config_dir_is_empty_or_missing { # @test fi } INNER - run "$BATS_WRAPPER" --tap "${TEST_TMPDIR}/read_config.bats" + run "$BATS_WRAPPER" --no-split --tap "${TEST_TMPDIR}/read_config.bats" assert_success assert_output --partial "ok 1" } @@ -52,23 +52,35 @@ function write_tmp { # @test rm -f /tmp/bats-wrapper-test-$$ } EOF - run "$BATS_WRAPPER" --tap "${TEST_TMPDIR}/write_tmp.bats" + run "$BATS_WRAPPER" --no-split --tap "${TEST_TMPDIR}/write_tmp.bats" assert_success } -function bats_wrapper_defaults_to_tap_output { # @test +function bats_wrapper_no_split_emits_tap_output { # @test cat >"${TEST_TMPDIR}/tap_default.bats" <<'EOF' #! /usr/bin/env bats function truth { # @test true } EOF - run "$BATS_WRAPPER" --no-sandbox "${TEST_TMPDIR}/tap_default.bats" + run "$BATS_WRAPPER" --no-split --no-sandbox "${TEST_TMPDIR}/tap_default.bats" assert_success # TAP output starts with version or plan line assert_line --index 0 --regexp "^(TAP version|1\.\.)" } +function bats_wrapper_full_output_is_alias_for_no_split { # @test + cat >"${TEST_TMPDIR}/alias.bats" <<'EOF' +#! /usr/bin/env bats +function truth { # @test + true +} +EOF + run "$BATS_WRAPPER" --full-output --no-sandbox "${TEST_TMPDIR}/alias.bats" + assert_success + assert_line --index 0 --regexp "^(TAP version|1\.\.)" +} + function bats_wrapper_no_sandbox_bypasses_fence { # @test cat >"${TEST_TMPDIR}/no_sandbox.bats" <<'EOF' #! /usr/bin/env bats @@ -88,7 +100,7 @@ function creates_file_in_tmpdir { # @test echo "marker" > "${BATS_TEST_TMPDIR}/marker.txt" } EOF - run "$BATS_WRAPPER" --no-tempdir-cleanup "${TEST_TMPDIR}/preserve.bats" + run "$BATS_WRAPPER" --no-split --no-tempdir-cleanup "${TEST_TMPDIR}/preserve.bats" assert_success assert_output --partial "ok 1" # Extract BATS_RUN_TMPDIR from output (printed by --no-tempdir-cleanup) @@ -101,7 +113,7 @@ EOF rm -rf "$bats_run_dir" } -function bats_wrapper_hide_passing_filters_ok_lines { # @test +function bats_wrapper_split_emits_failures_only_on_stdout { # @test cat >"${TEST_TMPDIR}/mixed.bats" <<'EOF' #! /usr/bin/env bats function passing_one { # @test @@ -114,45 +126,35 @@ function passing_two { # @test true } EOF - run "$BATS_WRAPPER" --hide-passing --no-sandbox "${TEST_TMPDIR}/mixed.bats" - # bats returns non-zero when tests fail + local pass_file="${TEST_TMPDIR}/passes.ndjson" + run "$BATS_WRAPPER" --no-sandbox --pass-out "$pass_file" "${TEST_TMPDIR}/mixed.bats" + # tap-dancer format-ndjson exits 1 when any record is a failure. assert_failure - assert_output --partial "not ok 2" - refute_line --regexp "^ok 1 " - refute_line --regexp "^ok 3 " + # The failure record names the failing test on stdout. + assert_output --partial "failing_one" + # Passing test names must not appear on stdout under split mode. + refute_output --partial "passing_one" + refute_output --partial "passing_two" } -function bats_wrapper_hide_passing_preserves_skip_and_todo { # @test - cat >"${TEST_TMPDIR}/directives.bats" <<'EOF' +function bats_wrapper_split_writes_passes_to_pass_out { # @test + cat >"${TEST_TMPDIR}/passes.bats" <<'EOF' #! /usr/bin/env bats -function skipped_test { # @test - skip "not ready" -} -function passing_test { # @test +function passing_a { # @test true } -EOF - run "$BATS_WRAPPER" --hide-passing --no-sandbox "${TEST_TMPDIR}/directives.bats" - assert_success - assert_line --regexp "^ok 1.* # [Ss][Kk][Ii][Pp]" - refute_line --regexp "^ok 2 " -} - -function bats_wrapper_hide_passing_preserves_yaml_on_failure { # @test - cat >"${TEST_TMPDIR}/yaml_fail.bats" <<'EOF' -#! /usr/bin/env bats -function passing_test { # @test +function passing_b { # @test true } -function failing_with_output { # @test - run echo "some diagnostic" - false -} EOF - run "$BATS_WRAPPER" --hide-passing --no-sandbox "${TEST_TMPDIR}/yaml_fail.bats" - assert_failure - assert_output --partial "not ok 2" - refute_line --regexp "^ok 1 " + local pass_file="${TEST_TMPDIR}/passes.ndjson" + run "$BATS_WRAPPER" --no-sandbox --pass-out "$pass_file" "${TEST_TMPDIR}/passes.bats" + assert_success + # The caller-supplied pass file should carry the passing records. + [ -f "$pass_file" ] + run cat "$pass_file" + assert_output --partial "passing_a" + assert_output --partial "passing_b" } # bats_wrapper_sandbox_denies_localhost_tcp_bind and @@ -165,18 +167,21 @@ EOF # seccomp deny set today. Re-add these tests once that gap is closed # (see https://github.com/amarbel-llc/bats/issues/3). -function bats_wrapper_hide_passing_preserves_plan_and_version { # @test +function bats_wrapper_no_split_restores_tap14_stream { # @test cat >"${TEST_TMPDIR}/plan.bats" <<'EOF' #! /usr/bin/env bats function passing_one { # @test true } -function passing_two { # @test - true +function failing_one { # @test + false } EOF - run "$BATS_WRAPPER" --hide-passing --no-sandbox "${TEST_TMPDIR}/plan.bats" - assert_success + run "$BATS_WRAPPER" --no-split --no-sandbox "${TEST_TMPDIR}/plan.bats" + assert_failure + # Plan + version + per-test ok/not ok records all present. assert_output --partial "1..2" + assert_line --regexp "^ok 1 " + assert_output --partial "not ok 2" assert_line --index 0 --regexp "^(TAP version|1\.\.)" } diff --git a/packages/batman/zz-tests_bats/ndjson_failure_demo.bats b/packages/batman/zz-tests_bats/ndjson_failure_demo.bats index 8d244f25f5..9a572cbc4f 100644 --- a/packages/batman/zz-tests_bats/ndjson_failure_demo.bats +++ b/packages/batman/zz-tests_bats/ndjson_failure_demo.bats @@ -2,8 +2,10 @@ # Artificial-failure demo suite for the `emitNdjson = true` batsLane path. # Intentionally contains one passing and one failing case so the produced -# `run.ndjson` exercises both `ok: true` and `ok: false` record shapes -# (per amarbel-llc/tap RFC 0001). +# NDJSON files exercise both `ok: true` and `ok: false` record shapes +# (per amarbel-llc/tap RFC 0001). Under the default `splitNdjson = true`, +# the failing case lands in `run.failures.ndjson` and the passing case +# lands in `run.passes.ndjson`. # # NOT picked up by `batman-self-proof` (which globs `*.bats` over this # same dir today). The new `batman-ndjson-demo` derivation in flake.nix