Skip to content

Commit 5cb030c

Browse files
brainrakeclaude
andauthored
fix(nix): fix linux-builder setup (#2669)
* fix(nix): add uid-range to darwin linux-builder Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(nix): pin qemu 10.2 and size darwin builder for uid-range tests Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * docs(nix): require official builder bootstrap for darwin setup Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix(nix): drop qemu pin from darwin builder Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * feat(nix): bootstrap official linux-builder in darwin setup Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 4c2d7e4 commit 5cb030c

3 files changed

Lines changed: 96 additions & 19 deletions

File tree

‎nix/docs/nixos-tests-on-macos.md‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,29 +12,31 @@ You need:
1212
Run the setup script to configure nix-darwin with the linux-builder:
1313

1414
```bash
15-
nix run .#setup-darwin-linux-builder
15+
sudo nix run .#setup-darwin-linux-builder
1616
```
1717

18-
Note that you don't have to checkout the repository to run the setup, you can run it directly from GitHub:
18+
The builder VM is a Linux system, and building it needs a Linux builder.
19+
If none is configured, the script first bootstraps with the official `nixos-26.05#darwin.linux-builder`, builds the configuration through it, and stops it before switching.
1920

20-
```bash
21-
nix run github:supabase/postgres#setup-darwin-linux-builder
22-
```
23-
24-
This command will:
21+
The script will:
2522
- Back up existing system files (`/etc/nix/nix.conf`, `/etc/bashrc`, `/etc/zshrc`)
2623
- Configure nix-darwin with the linux-builder VM
2724
- Install helper scripts for managing the builder
2825

2926
The linux-builder VM is configured with:
30-
- 6 CPU cores
31-
- 8GB RAM
32-
- 40GB disk
27+
- 8 CPU cores
28+
- 24GB RAM
29+
- 100GB disk
3330
- Support for both x86_64-linux and aarch64-linux builds
3431
- The `nixos-test` feature required for running NixOS tests
32+
- The `uid-range` feature for builds that request it
33+
- `auto-allocate-uids` and `cgroups` enabled inside the VM, so builds that need `uid-range` run
34+
- `-cpu host` passed to QEMU, since the default `-cpu max` breaks PAC-built guest binaries on M4
3535

3636
After setup completes, restart your shell to access the helper commands.
3737

38+
To re-apply the configuration after changing it, run the same command. The bootstrap is skipped once a builder exists.
39+
3840
## Verify the setup
3941

4042
The setup script runs verification automatically after configuration.

‎nix/hosts/darwin-nixostest/darwin-configuration.nix‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,24 +107,43 @@ in
107107
nix.linux-builder = {
108108
enable = true;
109109
ephemeral = true;
110-
maxJobs = 4;
110+
maxJobs = 2;
111111
supportedFeatures = [
112112
"kvm"
113113
"benchmark"
114114
"big-parallel"
115115
"nixos-test"
116+
"uid-range"
116117
];
117118
config = {
118119
virtualisation = {
119120
darwin-builder = {
120-
diskSize = 40 * 1024;
121-
memorySize = 8 * 1024;
121+
diskSize = 100 * 1024;
122+
memorySize = 24 * 1024;
122123
};
123-
cores = 6;
124+
cores = 8;
125+
};
126+
nix.settings = {
127+
experimental-features = [
128+
"auto-allocate-uids"
129+
"cgroups"
130+
];
131+
auto-allocate-uids = true;
132+
max-jobs = 2;
133+
cores = 4;
134+
system-features = [
135+
"kvm"
136+
"benchmark"
137+
"big-parallel"
138+
"nixos-test"
139+
"uid-range"
140+
];
124141
};
125142
};
126143
};
127144

145+
launchd.daemons.linux-builder.environment.QEMU_OPTS = "-cpu host";
146+
128147
nix.distributedBuilds = true;
129148

130149
system.stateVersion = 6;

‎nix/packages/setup-darwin-linux-builder.nix‎

Lines changed: 61 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,73 @@
11
{
22
inputs,
33
self,
4+
coreutils,
5+
gawk,
6+
gnugrep,
7+
openssh,
48
stdenv,
59
writeShellApplication,
610
}:
711
writeShellApplication {
812
name = "setup-darwin-linux-builder";
913
runtimeInputs = [
14+
coreutils
15+
gawk
16+
gnugrep
17+
openssh
1018
inputs.nix-darwin.packages.${stdenv.hostPlatform.system}.darwin-rebuild
1119
];
1220
text = ''
1321
set -euo pipefail
1422
23+
if [[ $EUID -ne 0 ]]; then
24+
echo "Run with sudo: sudo nix run .#setup-darwin-linux-builder" >&2
25+
exit 1
26+
fi
27+
28+
BOOTSTRAP_PID=""
29+
stop_bootstrap() {
30+
if [[ -n "$BOOTSTRAP_PID" ]]; then
31+
kill -- "-$BOOTSTRAP_PID" 2>/dev/null || true
32+
wait "$BOOTSTRAP_PID" 2>/dev/null || true
33+
BOOTSTRAP_PID=""
34+
fi
35+
}
36+
trap stop_bootstrap EXIT
37+
38+
if ! grep -qs aarch64-linux /etc/nix/machines; then
39+
echo "No Linux builder found. Bootstrapping with the official builder..."
40+
echo ""
41+
workdir=$(mktemp -d)
42+
cd "$workdir"
43+
set -m
44+
nix run github:NixOS/nixpkgs/nixos-26.05#darwin.linux-builder \
45+
>"$workdir/builder.log" 2>&1 </dev/null &
46+
BOOTSTRAP_PID=$!
47+
set +m
48+
49+
hostkey=""
50+
for _ in $(seq 1 300); do
51+
hostkey=$(ssh-keyscan -t ed25519 -p 31022 127.0.0.1 2>/dev/null |
52+
awk '!/^#/ {printf "%s %s", $2, $3}' | base64 | tr -d '\n') || true
53+
[[ -n "$hostkey" && -f "$workdir/keys/builder_ed25519" ]] && break
54+
hostkey=""
55+
sleep 1
56+
done
57+
if [[ -z "$hostkey" ]]; then
58+
echo "Official builder did not start. Log: $workdir/builder.log" >&2
59+
exit 1
60+
fi
61+
62+
echo "Building the configuration on the official builder..."
63+
NIX_SSHOPTS="-p 31022" nix build \
64+
"${self}#darwinConfigurations.darwin-nixostest.system" --no-link \
65+
--builders "ssh-ng://builder@127.0.0.1 aarch64-linux $workdir/keys/builder_ed25519 4 1 kvm,benchmark,big-parallel,nixos-test - $hostkey"
66+
stop_bootstrap
67+
cd /
68+
echo ""
69+
fi
70+
1571
echo "Configuring nix-darwin linux-builder..."
1672
echo ""
1773
@@ -20,7 +76,7 @@ writeShellApplication {
2076
for file in /etc/nix/nix.conf /etc/bashrc /etc/zshrc; do
2177
if [[ -f "$file" && ! -L "$file" ]]; then
2278
echo " Backing up $file"
23-
sudo mv "$file" "$file.before-nix-darwin"
79+
mv "$file" "$file.before-nix-darwin"
2480
fi
2581
done
2682
echo ""
@@ -29,21 +85,21 @@ writeShellApplication {
2985
for file in /etc/nix/nix.conf /etc/bashrc /etc/zshrc; do
3086
if [[ ! -L "$file" && -f "$file.before-nix-darwin" ]]; then
3187
echo " Restoring original $file"
32-
sudo mv "$file.before-nix-darwin" "$file"
88+
mv "$file.before-nix-darwin" "$file"
3389
fi
3490
done
3591
}
36-
trap revert ERR SIGINT SIGTERM
92+
trap 'stop_bootstrap; revert' ERR SIGINT SIGTERM
3793
3894
echo "This will configure your system with:"
3995
echo " - NixOS linux-builder VM (ephemeral)"
40-
echo " - 6 cores, 8GB RAM, 40GB disk"
96+
echo " - 8 cores, 24GB RAM, 100GB disk"
4197
echo " - Support for x86_64-linux and aarch64-linux builds"
4298
echo ""
4399
echo "Running darwin-rebuild switch..."
44100
echo ""
45101
46-
sudo darwin-rebuild switch --refresh --flake ${self}#darwin-nixostest
102+
darwin-rebuild switch --refresh --flake ${self}#darwin-nixostest
47103
48104
echo ""
49105
echo "Configuration complete!"

0 commit comments

Comments
 (0)