diff --git a/Gemfile.lock b/Gemfile.lock index 4b61b2fdf..8e4c36695 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -121,6 +121,9 @@ GEM path_expander (~> 2.0) prism (~> 1.7) sexp_processor (~> 4.8) + google-protobuf (4.35.1-arm64-darwin) + bigdecimal + rake (~> 13.3) google-protobuf (4.35.1-x86_64-linux-gnu) bigdecimal rake (~> 13.3) @@ -258,6 +261,7 @@ GEM simplecov (>= 0.22.0) tty-which (~> 0.5.0) virtus (~> 2.0) + rubydex (0.2.1-arm64-darwin) rubydex (0.2.1-x86_64-linux) sexp_processor (4.17.5) simplecov (0.22.0) @@ -272,6 +276,7 @@ GEM sorbet (0.6.13196) sorbet-static (= 0.6.13196) sorbet-runtime (0.6.13196) + sorbet-static (0.6.13196-universal-darwin) sorbet-static (0.6.13196-x86_64-linux) sorbet-static-and-runtime (0.6.13196) sorbet (= 0.6.13196) @@ -284,6 +289,7 @@ GEM rexml (>= 3.2.6) sorbet-static-and-runtime (>= 0.5.10187) thor (>= 0.19.2) + sqlite3 (2.9.5-arm64-darwin) sqlite3 (2.9.5-x86_64-linux-gnu) stackprof (0.2.28) stringio (3.2.0) @@ -301,6 +307,7 @@ GEM tsort thor (1.5.0) thread_safe (0.3.6) + tree_sitter (0.1.0-arm64-darwin) tree_sitter (0.1.0-x86_64-linux) tsort (0.2.0) tty-cursor (0.7.1) @@ -325,6 +332,8 @@ GEM zeitwerk (2.7.5) PLATFORMS + arm64-darwin + arm64-darwin-25 x86_64-linux-gnu DEPENDENCIES @@ -363,4 +372,4 @@ DEPENDENCIES tty-screen BUNDLED WITH - 2.7.2 + 4.0.9 diff --git a/clear b/clear index c447146a3..e36568a59 100755 --- a/clear +++ b/clear @@ -53,13 +53,26 @@ TRANSPILER = File.join(SRC_DIR, 'backends', 'transpiler.rb') TRANSPILER_CACHE_DIR = File.join(ZIG_DIR, '.clear-transpile-cache') # Find Zig compiler -ZIG = [ +REAL_ZIG = [ File.join(File.expand_path('~'), 'zig-x86_64-linux-0.16.0', 'zig'), File.join(ZIG_DIR, 'zig-new', 'zig'), File.join(ZIG_DIR, 'zig', 'zig'), `which zig 2>/dev/null`.strip ].find { |p| !p.empty? && File.exist?(p) } || 'zig' +# ...and cap its memory. A runaway zig reached 77 GB on a 96 GB machine and +# forced two physical resets; Darwin does not enforce RLIMIT_AS, so the cap has +# to be an external watchdog. Every zig spawn below goes through ZIG -- argv +# arrays and interpolated command strings alike -- so routing this one constant +# through the wrapper caps all of them. CLEAR_ZIG_MAX_RSS_MB tunes it. +ZIG_GUARD = File.join(CLEAR_ROOT, 'tools', 'zig-guard') +ZIG = if File.executable?(ZIG_GUARD) + ENV['CLEAR_REAL_ZIG'] = REAL_ZIG + ZIG_GUARD + else + REAL_ZIG + end + BUILD_SUPPORT_CONFIG = ClearBuildSupport::Config.new( src_dir: SRC_DIR, zig_dir: ZIG_DIR, diff --git a/compiler/ruby/ffi/c_header_importer.rb b/compiler/ruby/ffi/c_header_importer.rb index 454f6789a..45ded8e52 100644 --- a/compiler/ruby/ffi/c_header_importer.rb +++ b/compiler/ruby/ffi/c_header_importer.rb @@ -2,6 +2,7 @@ require "open3" require "set" +require "tmpdir" require "sorbet-runtime" # Expands the deliberately small, target-aware C header directive into the @@ -59,15 +60,27 @@ def self.import(header, library, source_dir:) # Ruby shells out through Open3. The self-hosted compiler maps this adapter # onto compilerZigTranslateC in the compiler native support module. # ruby-to-clear: skip + # Captured through FILES, not pipes. On macOS `zig translate-c` spins at + # 100% CPU forever when its stdout is a pipe; the identical command finishes + # in 0.06s when stdout is a file or /dev/null. Measured both ways on the + # same fixture and the same tmpdir, so it is the pipe and nothing else. + # Process.spawn (not system) so a missing zig still raises Errno::ENOENT for + # the caller's rescue to turn into "Zig is required to import C headers". sig { params(zig: String, source_dir: String, header_path: String).returns(String) } def self.compiler_zig_translate_c(zig, source_dir, header_path) - stdout, stderr, status = Open3.capture3(zig, "translate-c", "-I#{source_dir}", header_path) - unless status.success? - detail = stderr.split("\n").take(8).join("\n").strip - raise Error, "Zig could not import C header #{header_path.inspect}: #{detail}" - end + Dir.mktmpdir("clear-translate-c") do |dir| + out_path = File.join(dir, "stdout") + err_path = File.join(dir, "stderr") + pid = Process.spawn(zig, "translate-c", "-I#{source_dir}", header_path, + out: out_path, err: err_path) + _, status = Process.wait2(pid) + unless status.success? + detail = File.read(err_path).split("\n").take(8).join("\n").strip + raise Error, "Zig could not import C header #{header_path.inspect}: #{detail}" + end - stdout + File.read(out_path) + end end # ruby-to-clear: skip diff --git a/compiler/ruby/tools/pprof_converter.rb b/compiler/ruby/tools/pprof_converter.rb index 7fa6bb07e..88d682ffc 100644 --- a/compiler/ruby/tools/pprof_converter.rb +++ b/compiler/ruby/tools/pprof_converter.rb @@ -360,11 +360,33 @@ def self.parse_tabbed_columns(path, min_cols) # code where CLR markers don't exist, and we'd misattribute by # walking back through transpiled.zig anyway. (Repro: `pprof -list # entryWrapper` showed it pointing at random source.clear lines.) + # binutils spellings, in preference order. The resolver is OPTIONAL: macOS + # ships `atos` instead of binutils, so none of these exists there, and a + # profile converted on such a host must still emit its mappings with + # unresolved symbols rather than dying on ENOENT halfway through. + ADDR2LINE_CANDIDATES = %w[addr2line llvm-addr2line gaddr2line eu-addr2line].freeze + + # Try each spelling and take the first that runs. Spawning and rescuing is + # better than probing PATH first: it is one code path instead of two, and it + # cannot disagree with what exec would actually have found. + sig { params(addrs: Array, binary: String).returns(String) } + def self.addr2line_output(addrs, binary) + ADDR2LINE_CANDIDATES.each do |tool| + return IO.popen([tool, '-e', binary, '-f'] + addrs, err: '/dev/null', &:read) + rescue SystemCallError + next + end + '' + end + sig { params(addrs: Array, binary: T.nilable(String), profile_dir: String).returns(Hash) } def self.resolve_addrs(addrs, binary, profile_dir) return {} if addrs.empty? || binary.nil? - raw = IO.popen(['addr2line', '-e', binary, '-f'] + addrs, err: '/dev/null', &:read) - lines_out = raw.split("\n") + + # With no resolver at all, the empty output feeds the loop below, which + # already yields '?' for every field. Unresolved symbols are a degraded + # profile; a raised ENOENT is no profile at all. + lines_out = addr2line_output(addrs, binary).split("\n") zig_path = File.join(profile_dir, 'transpiled.zig') zig_lines = File.exist?(zig_path) ? File.readlines(zig_path) : nil diff --git a/compiler/spec/fmt_verifier_spec.rb b/compiler/spec/fmt_verifier_spec.rb index 8761c7bf0..ccfdf20a8 100644 --- a/compiler/spec/fmt_verifier_spec.rb +++ b/compiler/spec/fmt_verifier_spec.rb @@ -79,9 +79,15 @@ def write_cht(name, content = "FN main() RETURNS Void -> RETURN; END\n") expect(result.path).to eq("a.clear") end + # File.expand_path resolves a relative path against Dir.pwd, and Dir.pwd + # reports the SYMLINK-RESOLVED cwd. On macOS Dir.mktmpdir hands back + # /var/folders/... while /var is a symlink to /private/var, so the + # derived dir is the realpath. On Linux the two are already the same, + # which is why comparing against @tmp passed there. + expected_dir = File.realpath(@tmp) expect(calls).to eq([ - ["source", @tmp], - ["formatted", @tmp], + ["source", expected_dir], + ["formatted", expected_dir], ]) end diff --git a/compiler/spec/mem_guard_spec.rb b/compiler/spec/mem_guard_spec.rb new file mode 100644 index 000000000..cb7394e9e --- /dev/null +++ b/compiler/spec/mem_guard_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# tools/mem-guard and tools/zig-guard are shell, so their tests are shell too +# (tools/test/mem_guard_test.sh). This wrapper is what puts them in the Ruby +# suite, so a broken guard fails CI rather than waiting to be noticed the next +# time a zig run tries to take the machine down. +# +# The slow end-to-end case -- a real `clear build` proving the ZIG constant +# routes through the wrapper -- runs only with SLOW=1. +require 'open3' + +RSpec.describe 'tools/mem-guard and tools/zig-guard' do + it 'passes its behavioural suite with full line coverage' do + root = File.expand_path('../..', __dir__) + script = File.join(root, 'tools', 'test', 'mem_guard_test.sh') + out, status = Open3.capture2e({ 'COVERAGE' => '1' }, script, chdir: root) + + aggregate_failures do + expect(status).to be_success, "guard suite failed:\n#{out}" + expect(out).to match(/mem-guard\s+\d+\/\d+ lines = 100\.0%/), "mem-guard not fully covered:\n#{out}" + expect(out).to match(/zig-guard\s+\d+\/\d+ lines = 100\.0%/), "zig-guard not fully covered:\n#{out}" + expect(out).to include('0 failed') + end + end +end diff --git a/compiler/spec/semantic_equivalence_integration_spec.rb b/compiler/spec/semantic_equivalence_integration_spec.rb index 329dab6a4..fb1235ddf 100644 --- a/compiler/spec/semantic_equivalence_integration_spec.rb +++ b/compiler/spec/semantic_equivalence_integration_spec.rb @@ -1,6 +1,7 @@ require 'fileutils' require 'open3' require 'tmpdir' +require 'timeout' require_relative '../ruby/backends/transpiler' unless defined?(ZigTranspiler) require_relative '../../tools/fuzz/semantic_equivalence' @@ -11,6 +12,33 @@ 'MIRLowering', 'ZigTranspiler' ] do + ZIG_RUN_TIMEOUT = Integer(ENV.fetch('SEMANTIC_ZIG_TIMEOUT', '300')) + + # Captured through a FILE, not a pipe. macOS zig spins or wedges when its + # output is a pipe -- `zig translate-c` burns 100% CPU forever, and a `zig + # test` binary that panics blocks at 0% CPU writing its trace. Redirecting + # to a file costs nothing and sidesteps both. + def run_zig_to_file(*args, chdir: nil) + Dir.mktmpdir('semantic-zig-out') do |out_dir| + out_path = File.join(out_dir, 'combined') + opts = { out: out_path, err: %i[child out] } + opts[:chdir] = chdir if chdir + pid = Process.spawn(*args, **opts) + # A zig test binary that crashes can wedge in its own signal handler and + # sit at 0% CPU forever. Bound it: a visible failure with the captured + # output beats a suite that never finishes. + status = nil + begin + Timeout.timeout(ZIG_RUN_TIMEOUT) { _, status = Process.wait2(pid) } + rescue Timeout::Error + Process.kill('KILL', pid) + Process.wait(pid) + return [File.read(out_path) + "\n[timed out after #{ZIG_RUN_TIMEOUT}s]", nil] + end + [File.read(out_path), status] + end + end + def semantic_zig candidates = [ File.join(File.expand_path('~'), 'zig-x86_64-linux-0.16.0', 'zig'), @@ -58,10 +86,10 @@ def link_semantic_runtime(build_dir) end File.write(zig_path, ([header] + blocks).join("\n")) - format_output, format_status = Open3.capture2e(semantic_zig, 'fmt', zig_path) + format_output, format_status = run_zig_to_file(semantic_zig, 'fmt', zig_path) expect(format_status).to be_success, format_output - output, status = Open3.capture2e( + output, status = run_zig_to_file( semantic_zig, 'test', 'semantic-mutant.zig', 'runtime/switch.S', 'runtime/onRoot.S', '-lc', chdir: build_dir diff --git a/docs/agents/macos-arm64-port.md b/docs/agents/macos-arm64-port.md new file mode 100644 index 000000000..e164785ce --- /dev/null +++ b/docs/agents/macos-arm64-port.md @@ -0,0 +1,77 @@ +# macOS / arm64 port: what works, and what does not + +CLEAR builds and runs natively on **arm64 macOS** and on **aarch64-linux** as +of 2026-09-20. It previously built only for x86_64 Linux. + +## Build with `--safe` or `--optimized` on macOS + +The default self-hosted zig backend grows without bound there -- past 31 GB on +hello world. `tools/mem-guard` caps it; see below. + +## What the port fixed + +- `switch.S` / `onRoot.S` carry `#if defined(__aarch64__)` blocks that had + never been assembled: on aarch64 `#` introduces an immediate rather than a + comment, and `.section .text` is ELF-only. A `SYM()` macro handles Mach-O's + leading-underscore convention. +- The aarch64 `switch_context_asm` used the WRONG offsets -- `fiber-core.zig` + packs `x19` at `0x08`, the assembly stored it at `0x10` -- and never saved or + restored `x29`/`x30`. aarch64's `ret` branches to the link register where + x86's pops the stack, so the first switch into a fiber returned into the + `0xCC` Debug stack fill: `Segmentation fault at address 0xcccccccccccccccc`. + Fibers now seed `.lr` on aarch64. +- `runtime/io-backend.zig` selects the completion ring at comptime: real + io_uring on Linux, a `poll()`-based `PollRing` over a wake pipe elsewhere + (Darwin has no eventfd). Not an async backend -- socket submissions report + `AsyncIoUnsupported`. A kqueue backend drops in behind that interface. + +x86_64-linux is provably unaffected: `switch.S` and `onRoot.S` assembled for it +are instruction-for-instruction identical to before, with identical symbol +tables. + +## Two macOS defects, both fixed + +**A fiber with a null frame pointer sent the stack walker to address 0x8.** +aarch64 walkers follow the frame-pointer chain, reading the saved FP at [fp] +and the saved LR at [fp + 8], so a frame with fp = 0 dereferences 0x8. Fibers +seeded exactly that. Nothing hit it until the walk actually happened, which is +whenever something captures a stack trace -- and Zig's DebugAllocator, which +`zig test` uses as std.testing.allocator, does so on EVERY allocation. The +binary segfaulted and then wedged at 0% CPU in its own signal handler. A fiber +now parks a null frame record { saved_fp = 0, saved_lr = 0 } above its initial +SP and points fp at it, so the walk reads lr = 0 and stops. + +Note the misleading part: the same CLEAR source runs clean through +`./clear run --safe`, and ordinary fiber, stream and nested-BG programs all +pass. Only the debug allocator's stack capture triggered it. + +**`zig translate-c` spins at 100% CPU forever when its stdout is a PIPE.** Not +the input and not the directory -- the identical command on the identical +fixture in the identical tmpdir finishes in 0.087s from a shell. Measured from +Ruby on the same inputs: + +``` +system(..., out: File::NULL) 0.06s +system(..., out: ) 0.06s, 17675 bytes of correct output +IO.popen(...) (stdout = pipe) never returns, 99% CPU +``` + +`c_header_importer` and the semantic-equivalence spec now capture zig's output +through files. If you add a new zig invocation, do the same. + +## Adding CFI did not fix the unwinder hang + +Worth recording because it looked obvious: the aarch64 assembly had no +`.cfi_startproc`/`.cfi_endproc` while every x86-64 body had them. Adding it did +NOT fix the crash -- it reproduced with an identical stack. The CFI is in the +tree as correctness hygiene; the actual cause was the null frame pointer above. + +## zig can take the machine down; it is capped + +A single zig process reached **77.5 GB** on a 96 GB Mac and forced two physical +resets. Darwin does not enforce `RLIMIT_AS` -- `ulimit -v` is rejected outright +and a test process allocated 10 GB straight through it -- so the cap is an +external watchdog. `tools/mem-guard` polls a command's process tree and kills +it; `tools/zig-guard` resolves the real zig and delegates; `clear` routes its +`ZIG` constant through the guard, covering every spawn site. Default cap is a +quarter of RAM, tuned by `CLEAR_ZIG_MAX_RSS_MB` (`0` disables). diff --git a/tools/fuzz/hostile_frontend.rb b/tools/fuzz/hostile_frontend.rb index 3ef216af7..9ec381134 100644 --- a/tools/fuzz/hostile_frontend.rb +++ b/tools/fuzz/hostile_frontend.rb @@ -26,16 +26,38 @@ def initialize(timeout:, memory_mb:) @memory_bytes = memory_mb * 1024 * 1024 end - def run(source) - reader, writer = IO.pipe - pid = Process.spawn( - { "HOSTILE_FRONTEND_PARSE_TIMEOUT" => @timeout.to_s }, - RbConfig.ruby, __FILE__, "--worker", + # Darwin accepts RLIMIT_AS as a constant but setrlimit REJECTS it with + # EINVAL -- the address-space cap is simply not enforced there, which is the + # same reason `ulimit -v` is rejected on that platform. Learn it once by + # trying, rather than hardcoding a platform list that would go stale. + @rlimit_as_usable = true + + class << self + attr_accessor :rlimit_as_usable + end + + def spawn_worker(reader) + env = { "HOSTILE_FRONTEND_PARSE_TIMEOUT" => @timeout.to_s } + opts = { in: reader, out: File::NULL, err: File::NULL, - rlimit_as: @memory_bytes, # CPU time, not wall time, so a contended runner does not spend it. rlimit_cpu: [@timeout.ceil + 2, @timeout.ceil + 2], - ) + } + if self.class.rlimit_as_usable + begin + return Process.spawn(env, RbConfig.ruby, __FILE__, "--worker", **opts, rlimit_as: @memory_bytes) + rescue Errno::EINVAL + # Without the memory ceiling the CPU-time limit and the wall timeout + # still bound a hostile input; losing the spawn entirely does not. + self.class.rlimit_as_usable = false + end + end + Process.spawn(env, RbConfig.ruby, __FILE__, "--worker", **opts) + end + + def run(source) + reader, writer = IO.pipe + pid = spawn_worker(reader) reader.close writer.binmode writer.write(source) diff --git a/tools/mem-guard b/tools/mem-guard new file mode 100755 index 000000000..7a99d599f --- /dev/null +++ b/tools/mem-guard @@ -0,0 +1,73 @@ +#!/bin/sh +# Run any command under a hard resident-memory cap. +# +# Darwin does not enforce RLIMIT_AS -- `ulimit -v` is rejected outright and a +# test process allocated 10 GB straight through it -- so a runaway child cannot +# be capped by the kernel. This polls the child's process tree and kills it. +# A single zig run reached 77 GB here and forced two physical resets; the Ruby +# closure census is the same hazard from the other direction. +# +# MEM_GUARD_CAP_MB=32768 tools/mem-guard ruby big_job.rb +# +# MEM_GUARD_CAP_MB defaults to a quarter of RAM; 0 disables the watchdog. +# MEM_GUARD_LABEL names the child in messages. MEM_GUARD_REPORT=1 prints peak. +set -u + +[ $# -ge 1 ] || { echo "mem-guard: usage: mem-guard [args...]" >&2; exit 2; } +LABEL="${MEM_GUARD_LABEL:-$(basename "$1")}" + +CAP_MB="${MEM_GUARD_CAP_MB:-}" +if [ -z "$CAP_MB" ]; then + total_mb=$(( $(sysctl -n hw.memsize 2>/dev/null || echo 8589934592) / 1048576 )) + CAP_MB=$(( total_mb / 4 )) + [ "$CAP_MB" -lt 4096 ] && CAP_MB=4096 +fi +[ "$CAP_MB" = "0" ] && exec "$@" + +"$@" & +child=$! +cap_kb=$(( CAP_MB * 1024 )) +peak_kb=0 + +# Resident set of the WHOLE tree: a child that spawns workers (zig's linker, +# a fork pool) would slip a cap that only watched the root. +tree_kb() { + ps -Ao pid=,ppid=,rss= 2>/dev/null | awk -v root="$1" ' + { pid[NR]=$1; ppid[NR]=$2; rss[NR]=$3; n=NR } + END { + want[root]=1 + for (pass=0; pass<12; pass++) + for (i=1; i<=n; i++) if (want[ppid[i]]) want[pid[i]]=1 + t=0; for (i=1; i<=n; i++) if (want[pid[i]]) t+=rss[i] + print t+0 + }' +} + +kill_tree() { + ps -Ao pid=,ppid= 2>/dev/null | awk -v root="$1" ' + { pid[NR]=$1; ppid[NR]=$2; n=NR } + END { + want[root]=1 + for (pass=0; pass<12; pass++) + for (i=1; i<=n; i++) if (want[ppid[i]]) want[pid[i]]=1 + for (i=1; i<=n; i++) if (want[pid[i]]) print pid[i] + }' | while read -r p; do kill -9 "$p" 2>/dev/null; done +} + +trap 'kill_tree "$child"; exit 130' INT TERM + +while kill -0 "$child" 2>/dev/null; do + rss_kb=$(tree_kb "$child"); [ -n "$rss_kb" ] || rss_kb=0 + [ "$rss_kb" -gt "$peak_kb" ] && peak_kb=$rss_kb + if [ "$rss_kb" -gt "$cap_kb" ]; then + kill_tree "$child" + echo "mem-guard: KILLED $LABEL at $(( rss_kb / 1024 )) MB resident (cap ${CAP_MB} MB)." >&2 + echo "mem-guard: this is a memory runaway, not a compile error." >&2 + wait "$child" 2>/dev/null + exit 137 + fi + sleep 0.25 +done +wait "$child"; status=$? +[ -n "${MEM_GUARD_REPORT:-}" ] && echo "mem-guard: $LABEL peak $(( peak_kb / 1024 )) MB" >&2 +exit $status diff --git a/tools/test/mem_guard_test.sh b/tools/test/mem_guard_test.sh new file mode 100755 index 000000000..1ff4fd6e2 --- /dev/null +++ b/tools/test/mem_guard_test.sh @@ -0,0 +1,234 @@ +#!/bin/bash +# Behavioural tests for tools/mem-guard and tools/zig-guard. +# +# Run with COVERAGE=1 to trace executed lines (PS4 carries $LINENO) and print a +# per-script line-coverage report. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +GUARD="$ROOT/tools/mem-guard" +ZGUARD="$ROOT/tools/zig-guard" +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +pass=0; fail=0 + +ok() { if [ "$1" = "$2" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $3 (want '$2', got '$1')"; fi; } +contains() { case "$1" in *"$2"*) pass=$((pass+1));; *) fail=$((fail+1)); echo "FAIL: $3";; esac; } + +run() { # run