Skip to content

Commit 8c0cc12

Browse files
committed
fix: detect gpu for toolkit cdi preflight
1 parent d129ab7 commit 8c0cc12

5 files changed

Lines changed: 117 additions & 26 deletions

File tree

src/lib/onboard.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1646,11 +1646,11 @@ async function preflight(
16461646
device: preflightOpts.sandboxGpuDevice ?? null,
16471647
});
16481648
exitOnSandboxGpuConfigErrors(sandboxGpuConfig);
1649-
const optedOutGpuPassthrough =
1649+
const explicitlyOptedOutGpuPassthrough =
16501650
preflightOpts.optedOutGpuPassthrough === true ||
16511651
preflightOpts.noGpu === true ||
1652-
!sandboxGpuConfig.sandboxGpuEnabled;
1653-
assertCdiNvidiaGpuSpecPresent(host, optedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform);
1652+
sandboxGpuConfig.mode === "0";
1653+
assertCdiNvidiaGpuSpecPresent(host, explicitlyOptedOutGpuPassthrough, sandboxGpuConfig.hostGpuPlatform);
16541654

16551655
assertDockerBridgeAndContainerDnsHealthy(host, isNonInteractive());
16561656

src/lib/onboard/machine/handlers/preflight.test.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,13 +186,35 @@ describe("handlePreflightState", () => {
186186
expect(harness.deps.startRecordedStep).not.toHaveBeenCalled();
187187
expect(harness.deps.assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith(
188188
{ cdiNvidiaGpuSpecMissing: false },
189-
true,
189+
false,
190190
undefined,
191191
);
192192
expect(harness.deps.validateSandboxGpuPreflight).toHaveBeenCalledOnce();
193193
expect(result.resumePreflight).toBe(true);
194194
});
195195

196+
it("keeps CDI guard active on resume when auto mode disables GPU after failed detection", async () => {
197+
const session = createSession();
198+
session.steps.preflight.status = "complete";
199+
session.gpuPassthrough = false;
200+
const assertCdiNvidiaGpuSpecPresent = vi.fn();
201+
const host = { cdiNvidiaGpuSpecMissing: true };
202+
const harness = createDeps({
203+
detectGpu: vi.fn(() => null),
204+
getResumeSandboxGpuOverrides: vi.fn(() => ({ flag: null, device: null })),
205+
resolveSandboxGpuConfig,
206+
assessHost: () => host,
207+
assertCdiNvidiaGpuSpecPresent,
208+
});
209+
210+
await handlePreflightState({
211+
...baseOptions(harness.deps, session),
212+
resume: true,
213+
});
214+
215+
expect(assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith(host, false, null);
216+
});
217+
196218
it("passes host GPU platform into the resumed CDI guard", async () => {
197219
const session = createSession();
198220
session.steps.preflight.status = "complete";
@@ -220,7 +242,7 @@ describe("handlePreflightState", () => {
220242

221243
expect(assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith(
222244
{ cdiNvidiaGpuSpecMissing: false },
223-
true,
245+
false,
224246
"jetson",
225247
);
226248
});

src/lib/onboard/machine/handlers/preflight.ts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,9 +143,7 @@ export async function handlePreflightState<
143143
});
144144
deps.validateSandboxGpuPreflight(resumeSandboxGpuConfig);
145145
const resumeOptedOutGpuPassthrough =
146-
noGpu ||
147-
(!gpuRequested && session?.gpuPassthrough === false) ||
148-
!resumeSandboxGpuConfig.sandboxGpuEnabled;
146+
noGpu || effectiveSandboxGpuFlag === "disable" || resumeSandboxGpuConfig.mode === "0";
149147
const resumeHost = deps.assessHost();
150148
// Reject unsupported runtimes (Podman) BEFORE the CDI GPU-spec
151149
// backstop and the Docker-specific bridge/DNS probes so Podman

src/lib/onboard/preflight-cdi.test.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,50 @@ describe("assessHost — CDI", () => {
6767
expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
6868
});
6969

70+
it("plans toolkit bootstrap when PCI detects NVIDIA hardware but nvidia-smi and nvidia-ctk are absent", () => {
71+
const result = assessHost({
72+
platform: "linux",
73+
env: {},
74+
release: "6.8.0-58-generic",
75+
readFileImpl: (filePath: string) =>
76+
filePath.endsWith("other.yaml")
77+
? "cdiVersion: 0.5.0\nkind: vendor.example/device\ndevices: []\n"
78+
: "Linux version 6.8.0-58-generic",
79+
readdirImpl: (dir: string) => (dir === "/etc/cdi" ? ["other.yaml"] : []),
80+
runCaptureImpl: (command: readonly string[]) => {
81+
if (command.join(" ").includes("apt-get")) return "/usr/bin/apt-get";
82+
if (command[0] === "lspci") {
83+
return "01:00.0 VGA compatible controller: NVIDIA Corporation GA102 [GeForce RTX 3090]\n";
84+
}
85+
if (command[0] === "systemctl" && command[1] === "is-active") return "active";
86+
if (command[0] === "systemctl" && command[1] === "is-enabled") return "enabled";
87+
return "";
88+
},
89+
dockerInfoOutput: JSON.stringify({
90+
ServerVersion: "27.0",
91+
OperatingSystem: "Ubuntu 24.04",
92+
CDISpecDirs: ["/etc/cdi", "/var/run/cdi"],
93+
}),
94+
commandExistsImpl: (name: string) =>
95+
name === "docker" || name === "lspci" || name === "systemctl",
96+
});
97+
98+
expect(result.hasNvidiaGpu).toBe(true);
99+
expect(result.nvidiaContainerToolkitInstalled).toBe(false);
100+
expect(result.cdiNvidiaGpuSpecMissing).toBe(true);
101+
102+
const action = planHostRemediation(result).find(
103+
(entry: { id: string }) => entry.id === "install_nvidia_container_toolkit",
104+
);
105+
expect(action).toBeTruthy();
106+
expect(action?.blocking).toBe(true);
107+
expect(action?.commands).toContain("sudo apt-get install -y nvidia-container-toolkit");
108+
expect(action?.commands.some((command) => command.includes("nvidia-ctk cdi generate"))).toBe(
109+
true,
110+
);
111+
expect(action?.commands.some((command) => command.includes("nvidia-ctk cdi list"))).toBe(true);
112+
});
113+
70114
it("does not flag the host when an nvidia.com/gpu YAML spec is present", () => {
71115
const result = assessHost({
72116
platform: "linux",

src/lib/onboard/preflight.ts

Lines changed: 45 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -385,11 +385,29 @@ function isHeadlessLikely(env: NodeJS.ProcessEnv): boolean {
385385
return !env.DISPLAY && !env.WAYLAND_DISPLAY && !env.TERM_PROGRAM;
386386
}
387387

388-
function detectNvidiaGpu(runCaptureImpl: RunCaptureFn): boolean {
389-
if (!commandExists("nvidia-smi", runCaptureImpl)) {
390-
return false;
391-
}
392-
return Boolean(String(runCaptureImpl(["nvidia-smi", "-L"], { ignoreError: true }) || "").trim());
388+
function detectNvidiaGpu(opts: {
389+
platform: NodeJS.Platform | string;
390+
isWsl: boolean;
391+
runCaptureImpl: RunCaptureFn;
392+
commandExistsImpl?: (commandName: string) => boolean;
393+
}): boolean {
394+
const commandExistsImpl =
395+
opts.commandExistsImpl ??
396+
((commandName: string) => commandExists(commandName, opts.runCaptureImpl));
397+
if (commandExistsImpl("nvidia-smi")) {
398+
const smiOutput = opts.runCaptureImpl(["nvidia-smi", "-L"], { ignoreError: true });
399+
if (String(smiOutput || "").trim()) return true;
400+
}
401+
402+
if (opts.platform !== "linux" || opts.isWsl || !commandExistsImpl("lspci")) return false;
403+
const pciOutput = opts.runCaptureImpl(["lspci", "-nn"], { ignoreError: true });
404+
return String(pciOutput || "")
405+
.split(/\r?\n/)
406+
.some(
407+
(line) =>
408+
/nvidia/i.test(line) &&
409+
/(vga compatible controller|3d controller|display controller)/i.test(line),
410+
);
393411
}
394412

395413
function detectPackageManager(runCaptureImpl: RunCaptureFn): PackageManager {
@@ -456,12 +474,33 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment {
456474
runCapture(command, { ignoreError: options?.ignoreError ?? false }));
457475
const readFileImpl = opts.readFileImpl ?? fs.readFileSync;
458476
const readdirImpl = opts.readdirImpl ?? ((dir: string) => fs.readdirSync(dir));
477+
const shouldReadLinuxHostDetails = platform === "linux";
478+
const release = opts.release ?? (shouldReadLinuxHostDetails ? os.release() : "");
479+
const procVersion =
480+
opts.procVersion ??
481+
(shouldReadLinuxHostDetails
482+
? (() => {
483+
try {
484+
return readFileImpl("/proc/version", "utf-8");
485+
} catch {
486+
return "";
487+
}
488+
})()
489+
: "");
490+
const isWslHost = detectWsl({ platform, env, release, procVersion });
459491
const dockerInstalled =
460492
opts.commandExistsImpl?.("docker") ?? commandExists("docker", runCaptureImpl);
461493
const nodeInstalled = opts.commandExistsImpl?.("node") ?? commandExists("node", runCaptureImpl);
462494
const openshellInstalled =
463495
opts.commandExistsImpl?.("openshell") ?? commandExists("openshell", runCaptureImpl);
464-
const hasNvidiaGpu = opts.gpuProbeImpl?.() ?? detectNvidiaGpu(runCaptureImpl);
496+
const hasNvidiaGpu =
497+
opts.gpuProbeImpl?.() ??
498+
detectNvidiaGpu({
499+
platform,
500+
isWsl: isWslHost,
501+
runCaptureImpl,
502+
commandExistsImpl: opts.commandExistsImpl,
503+
});
465504
const nvidiaContainerToolkitInstalled =
466505
opts.commandExistsImpl?.("nvidia-ctk") ?? commandExists("nvidia-ctk", runCaptureImpl);
467506
const packageManager = detectPackageManager(runCaptureImpl);
@@ -480,22 +519,10 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment {
480519
dockerReachable = true;
481520
dockerRunning = true;
482521
}
483-
484-
const release = opts.release ?? os.release();
485-
const procVersion =
486-
opts.procVersion ??
487-
(() => {
488-
try {
489-
return readFileImpl("/proc/version", "utf-8");
490-
} catch {
491-
return "";
492-
}
493-
})();
494522
let runtime = inferContainerRuntime(dockerInfoOutput);
495523
if (dockerReachable && runtime === "unknown" && platform === "linux") {
496524
runtime = "docker";
497525
}
498-
const isWslHost = detectWsl({ platform, env, release, procVersion });
499526
const dockerCgroupVersion = dockerReachable
500527
? parseDockerCgroupVersion(dockerInfoOutput)
501528
: "unknown";

0 commit comments

Comments
 (0)