feat(remote): add rocm remote — serve a model on a remote GPU host over SSH

apps/rocm/src/main.rs

@@ -9,6 +9,7 @@ mod dash;
 mod dash_seam;
 mod provider_keys;
 mod providers;
+mod remote;
 mod serve_summary;
 mod therock;
 mod uninstall;
@@ -331,6 +332,11 @@ rocm serve qwen2.5-7b-instruct --verbose --device gpu_required")]
         #[command(subcommand)]
         command: Option<ServicesCommand>,
     },
+    /// Serve a model on a remote GPU host and forward it to this machine.
+    Remote {
+        #[command(subcommand)]
+        command: remote::RemoteCommand,
+    },
     /// Manage optional background checks and review requests.
     Automations {
         #[command(subcommand)]
@@ -602,6 +608,9 @@ enum ServicesCommand {
         /// Include failed, stopped, and old service records.
         #[arg(short, long)]
         all: bool,
+        /// Emit the machine-readable service records as JSON.
+        #[arg(long)]
+        json: bool,
     },
     /// Show logs for a local model server.
     Logs {
@@ -1505,6 +1514,7 @@ fn dispatch(cli: Cli) -> Result<()> {
         }),
         Some(Command::Comfyui { command }) => comfyui(command),
         Some(Command::Services { command }) => services(command),
+        Some(Command::Remote { command }) => remote::remote(command),
         Some(Command::Automations { command }) => automations(command),
         Some(Command::Config { command }) => config(command),
         Some(Command::Logs {
@@ -4444,9 +4454,20 @@ fn run_foreground_service(
 
 fn services(command: Option<ServicesCommand>) -> Result<()> {
     let paths = AppPaths::discover()?;
-    match command.unwrap_or(ServicesCommand::List { all: false }) {
-        ServicesCommand::List { all } => {
-            print!("{}", render_services_text(&paths, all)?);
+    match command.unwrap_or(ServicesCommand::List {
+        all: false,
+        json: false,
+    }) {
+        ServicesCommand::List { all, json } => {
+            if json {
+                let records = load_managed_services(&paths)?
+                    .into_iter()
+                    .filter(|record| all || managed_service_is_live(record))
+                    .collect::<Vec<_>>();
+                println!("{}", serde_json::to_string_pretty(&records)?);
+            } else {
+                print!("{}", render_services_text(&paths, all)?);
+            }
             Ok(())
         }
         ServicesCommand::Logs { service_id } => {
@@ -15446,6 +15467,7 @@ fn treat_as_natural_language(args: &[String]) -> bool {
         "comfyui",
         "comfy",
         "services",
+        "remote",
         "automations",
         "config",
         "logs",
@@ -16246,6 +16268,21 @@ mod tests {
         assert!(!should_treat_as_freeform(&structured));
     }
 
+    #[test]
+    fn remote_subcommand_routes_to_clap_not_the_planner() {
+        // `rocm remote serve …` must reach the structured command, not be
+        // swallowed by the natural-language planner. Guards the STRUCTURED
+        // allowlist against dropping `remote`.
+        let invocation = parse_freeform_invocation(&[
+            "remote".to_owned(),
+            "serve".to_owned(),
+            "gpu-box".to_owned(),
+            "qwen2.5-7b-instruct".to_owned(),
+        ]);
+        assert!(!treat_as_natural_language(&invocation.request_args));
+        assert!(!should_treat_as_freeform(&invocation));
+    }
+
     #[test]
     fn freeform_invocation_rejects_unquoted_structured_command_names_after_yes() {
         let invalid_install = parse_freeform_invocation(&[
@@ -19063,6 +19100,56 @@ install therock";
         Ok(())
     }
 
+    #[test]
+    fn services_list_json_round_trips_records() -> Result<()> {
+        // `rocm services list --json` is the machine-readable source of truth
+        // that `rocm remote` reconciliation reads over SSH. Guard that the
+        // emitted JSON deserializes back into ManagedServiceRecord with the
+        // fields consumers depend on (service_id, endpoint_url, port, status).
+        let (root, paths) = test_paths("services-list-json");
+        paths.ensure()?;
+        let current_pid = std::process::id();
+        for (service_id, status, port) in [
+            ("svc-a", "ready", 11440_u16),
+            ("svc-b", "failed", 11441_u16),
+        ] {
+            let mut record = ManagedServiceRecord::new(
+                &paths,
+                service_id,
+                "pytorch",
+                "qwen",
+                "Qwen/Qwen3.5",
+                "127.0.0.1",
+                port,
+                "managed",
+                current_pid,
+                Some("therock-release".to_owned()),
+                None,
+                Some("gpu_required".to_owned()),
+            );
+            record.status = status.to_owned();
+            record.write()?;
+        }
+
+        // Mirror the handler's `--json --all` path: serialize every record.
+        let records = load_managed_services(&paths)?;
+        let json = serde_json::to_string_pretty(&records)?;
+        let parsed: Vec<ManagedServiceRecord> = serde_json::from_str(&json)?;
+        let _ = fs::remove_dir_all(root);
+
+        assert_eq!(parsed.len(), 2);
+        let a = parsed
+            .iter()
+            .find(|r| r.service_id == "svc-a")
+            .expect("svc-a present in JSON");
+        assert_eq!(a.port, 11440);
+        assert_eq!(a.engine, "pytorch");
+        assert!(!a.status.is_empty());
+        assert!(a.endpoint_url.contains("11440"));
+        assert!(parsed.iter().any(|r| r.service_id == "svc-b"));
+        Ok(())
+    }
+
     #[test]
     fn render_services_text_demotes_stale_ready_record() -> Result<()> {
         let (root, paths) = test_paths("services-stale-ready");

apps/rocm/src/remote/bootstrap.rs

@@ -0,0 +1,244 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier: MIT
+
+//! Ensuring a remote host is ready to serve: probe for the `rocm` CLI and ROCm,
+//! and push the CLI if it is missing.
+//!
+//! Scope decisions (see `plans/rocm-remote-serve.md`):
+//! - **CLI missing → auto-push.** The whole `rocm` runtime, including the managed
+//!   daemon, is a single self-contained binary (`rocm daemon` is the supervisor;
+//!   there is no separate `rocmd` to install), so making the remote ready is just
+//!   copying one file. Because the release repo is currently private (no public
+//!   `curl` URL), v1 pushes the *local* `rocm` binary over `scp`. This works when
+//!   the local machine is itself linux-amd64 (the demo case); other hosts get a
+//!   clear error pointing at the manual install.
+//! - **ROCm missing → detect and fail.** We never attempt to install the
+//!   multi-GB ROCm SDK remotely; we report it clearly and stop.
+
+use anyhow::{Context, Result, bail};
+
+use super::transport::Transport;
+
+/// Where a freshly-pushed CLI lands on the remote. Chosen to match the
+/// documented manual install location (`~/.local/bin`).
+const REMOTE_CLI_PATH: &str = "~/.local/bin/rocm";
+
+/// What a probe found on the remote host.
+#[derive(Debug, Clone)]
+pub struct RemoteReadiness {
+    pub cli_present: bool,
+    pub cli_version: Option<String>,
+    pub rocm_present: bool,
+}
+
+/// Probe the remote for the `rocm` CLI and a ROCm installation. Neither absence
+/// is an error here — that judgement belongs to [`ensure_ready`].
+pub fn probe(transport: &dyn Transport) -> Result<RemoteReadiness> {
+    let cli = transport.exec("rocm --version")?;
+    let (cli_present, cli_version) = if cli.success {
+        let version = cli.stdout.lines().next().map(|l| l.trim().to_owned());
+        (true, version.filter(|v| !v.is_empty()))
+    } else {
+        (false, None)
+    };
+
+    // ROCm detection independent of our CLI: any of the usual markers is enough.
+    let rocm = transport.exec(
+        "command -v rocminfo >/dev/null 2>&1 || command -v amd-smi >/dev/null 2>&1 || test -d /opt/rocm",
+    )?;
+
+    Ok(RemoteReadiness {
+        cli_present,
+        cli_version,
+        rocm_present: rocm.success,
+    })
+}
+
+/// How to invoke `rocm` on the remote once we know it is present — either the
+/// CLI already on `PATH`, or the explicit path we just installed to.
+#[derive(Debug, Clone)]
+pub struct RemoteCli {
+    invocation: String,
+}
+
+impl RemoteCli {
+    /// The command prefix to use when building remote `rocm …` invocations.
+    pub fn invocation(&self) -> &str {
+        &self.invocation
+    }
+}
+
+/// Make the remote ready to serve, pushing the CLI if needed. Returns how to
+/// invoke `rocm` on the remote. Fails (without attempting any install) if ROCm
+/// is absent.
+pub fn ensure_ready(transport: &dyn Transport) -> Result<RemoteCli> {
+    let readiness = probe(transport)?;
+
+    if !readiness.rocm_present {
+        bail!(
+            "ROCm was not detected on the remote host (no rocminfo/amd-smi and no /opt/rocm).\n\
+             Install ROCm on the remote first, then re-run. `rocm remote serve` does not \
+             install the ROCm SDK for you."
+        );
+    }
+
+    if readiness.cli_present {
+        match readiness.cli_version.as_deref() {
+            Some(version) => println!("  remote rocm: {version}"),
+            None => println!("  remote rocm: present"),
+        }
+        return Ok(RemoteCli {
+            invocation: "rocm".to_owned(),
+        });
+    }
+
+    // CLI absent: push the local binary.
+    println!("  remote rocm: not found — installing the local rocm binary...");
+    push_local_cli(transport)?;
+    println!("  remote rocm: installed to {REMOTE_CLI_PATH}");
+    Ok(RemoteCli {
+        invocation: REMOTE_CLI_PATH.to_owned(),
+    })
+}
+
+/// Copy the locally-running `rocm` binary to the remote and verify it runs.
+fn push_local_cli(transport: &dyn Transport) -> Result<()> {
+    let local_exe = std::env::current_exe()
+        .context("failed to resolve the local rocm executable to push to the remote")?;
+    let size_hint = std::fs::metadata(&local_exe)
+        .ok()
+        .map(|m| format!(" (~{} MB)", m.len() / 1_000_000))
+        .unwrap_or_default();
+    println!("  copying {}{size_hint} ...", local_exe.display());
+
+    transport
+        .run("mkdir -p \"$HOME/.local/bin\"")
+        .context("failed to create ~/.local/bin on the remote")?;
+    transport.push_file(&local_exe, REMOTE_CLI_PATH).context(
+        "failed to copy the local rocm binary to the remote. If your local machine is not \
+             linux-amd64, install rocm on the remote manually and re-run.",
+    )?;
+    transport
+        .run("chmod +x \"$HOME/.local/bin/rocm\"")
+        .context("failed to mark the pushed rocm binary executable")?;
+
+    // Verify the pushed binary actually runs on the remote (catches arch/libc
+    // mismatch early, before we try to serve with it).
+    let verify = transport.exec("\"$HOME/.local/bin/rocm\" --version")?;
+    if !verify.success {
+        bail!(
+            "pushed rocm binary does not run on the remote (exit {}): {}\n\
+             This usually means an architecture or libc mismatch. Install rocm on the remote \
+             manually and re-run.",
+            verify
+                .code
+                .map_or_else(|| "signal".to_owned(), |c| c.to_string()),
+            verify.stderr.trim(),
+        );
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::remote::transport::RemoteOutcome;
+    use std::cell::RefCell;
+
+    /// A scripted transport: each `exec` pops the next queued outcome and records
+    /// the command it was asked to run. `run` reuses the trait default over
+    /// `exec`, so queued outcomes must account for `run` calls too.
+    struct ScriptedTransport {
+        outcomes: RefCell<Vec<RemoteOutcome>>,
+        commands: RefCell<Vec<String>>,
+        pushed: RefCell<Vec<String>>,
+    }
+
+    impl ScriptedTransport {
+        fn new(outcomes: Vec<RemoteOutcome>) -> Self {
+            Self {
+                outcomes: RefCell::new(outcomes),
+                commands: RefCell::new(Vec::new()),
+                pushed: RefCell::new(Vec::new()),
+            }
+        }
+    }
+
+    fn ok(stdout: &str) -> RemoteOutcome {
+        RemoteOutcome {
+            success: true,
+            code: Some(0),
+            stdout: stdout.to_owned(),
+            stderr: String::new(),
+        }
+    }
+
+    fn fail() -> RemoteOutcome {
+        RemoteOutcome {
+            success: false,
+            code: Some(1),
+            stdout: String::new(),
+            stderr: "not found".to_owned(),
+        }
+    }
+
+    impl Transport for ScriptedTransport {
+        fn exec(&self, command: &str) -> Result<RemoteOutcome> {
+            self.commands.borrow_mut().push(command.to_owned());
+            Ok(self
+                .outcomes
+                .borrow_mut()
+                .drain(..1)
+                .next()
+                .unwrap_or_else(fail))
+        }
+        fn push_file(&self, _local: &std::path::Path, remote_path: &str) -> Result<()> {
+            self.pushed.borrow_mut().push(remote_path.to_owned());
+            Ok(())
+        }
+        fn open_detached_forward(
+            &self,
+            _local_port: u16,
+            _remote_host: &str,
+            _remote_port: u16,
+        ) -> Result<u32> {
+            unreachable!("bootstrap tests do not forward")
+        }
+    }
+
+    #[test]
+    fn probe_reports_present_cli_and_rocm() {
+        let t = ScriptedTransport::new(vec![ok("rocm 0.3.0"), ok("")]);
+        let r = probe(&t).unwrap();
+        assert!(r.cli_present);
+        assert_eq!(r.cli_version.as_deref(), Some("rocm 0.3.0"));
+        assert!(r.rocm_present);
+    }
+
+    #[test]
+    fn ensure_ready_bails_when_rocm_absent() {
+        // cli --version fails, rocm detection fails.
+        let t = ScriptedTransport::new(vec![fail(), fail()]);
+        let err = ensure_ready(&t).unwrap_err().to_string();
+        assert!(err.contains("ROCm was not detected"), "got: {err}");
+    }
+
+    #[test]
+    fn ensure_ready_returns_path_invocation_after_push() {
+        // probe: cli absent (fail), rocm present (ok);
+        // push_local_cli: mkdir ok, chmod ok, verify ok.
+        let t = ScriptedTransport::new(vec![fail(), ok(""), ok(""), ok(""), ok("rocm 0.3.0")]);
+        let cli = ensure_ready(&t).unwrap();
+        assert_eq!(cli.invocation(), REMOTE_CLI_PATH);
+        assert!(t.pushed.borrow().iter().any(|p| p == REMOTE_CLI_PATH));
+    }
+
+    #[test]
+    fn ensure_ready_uses_path_rocm_when_present() {
+        let t = ScriptedTransport::new(vec![ok("rocm 0.3.0"), ok("")]);
+        let cli = ensure_ready(&t).unwrap();
+        assert_eq!(cli.invocation(), "rocm");
+        assert!(t.pushed.borrow().is_empty());
+    }
+}