Add support for Hyperlight KVM guest debugging using gdb

.github/workflows/dep_rust.yml

@@ -87,6 +87,7 @@ jobs:
           # make sure certain cargo features compile
           cargo check -p hyperlight-host --features crashdump
           cargo check -p hyperlight-host --features print_debug
+          cargo check -p hyperlight-host --features gdb
 
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}

docs/README.md

@@ -34,6 +34,7 @@ This project is composed internally of several internal components, depicted in
 
 * [Security guidance for developers](./security-guidance-for-developers.md)
 * [Paging Development Notes](./paging-development-notes.md)
+* [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md)
 * [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md)
 * [How to make a Hyperlight release](./how-to-make-releases.md)
 * [Getting Hyperlight Metrics, Logs, and Traces](./hyperlight-metrics-logs-and-traces.md)

docs/how-to-debug-a-hyperlight-guest.md

@@ -0,0 +1,41 @@
+# How to debug a Hyperlight guest
+
+Hyperlight supports gdb debugging of a guest running inside a Hyperlight sandbox.
+When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight sandbox can be configured
+to start listening for a gdb connection.
+
+## Example
+The snipped of a rust host application below configures the Hyperlight Sandbox to
+listen on port `9050` for a gdb client to connect.
+
+```rust
+    let mut cfg = SandboxConfiguration::default();
+    cfg.set_guest_debug_port(9050);
+
+    // Create an uninitialized sandbox with a guest binary
+    let mut uninitialized_sandbox = UninitializedSandbox::new(
+        hyperlight_host::GuestBinary::FilePath(
+            hyperlight_testing::simple_guest_as_string().unwrap(),
+        ),
+        Some(cfg), // configuration
+        None, // default run options
+        None, // default host print function
+    )?;
+```
+
+The execution of the guest will wait for gdb to attach.
+
+One can use a simple gdb config to provide the symbols and desired configuration:
+
+For the above snippet, the below contents of the `.gdbinit` file can be used to
+provide configuration to gdb startup.
+```gdb
+file path/to/symbols.elf
+target remote :9050
+set disassembly-flavor intel
+set disassemble-next-line on
+enable pretty-printer
+layout src
+```
+
+One can find more information about the `.gdbinit` file at [gdbinit(5)](https://www.man7.org/linux/man-pages/man5/gdbinit.5.html).

src/hyperlight_host/Cargo.toml

@@ -71,6 +71,8 @@ sha256 = "1.4.0"
 windows-version = "0.1"
 
 [target.'cfg(unix)'.dependencies]
+gdbstub = "0.7.3"
+gdbstub_arch = "0.3.1"
 seccompiler = { version = "0.4.0", optional = true }
 kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true }
 kvm-ioctls = { version = "0.20", optional = true }
@@ -128,6 +130,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 inprocess = []
+# This enables compilation of gdb stub for easy debug in the guest
+gdb = []
 
 [[bench]]
 name = "benchmarks"

src/hyperlight_host/build.rs

@@ -89,6 +89,7 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
+        gdb: { all(feature = "gdb", debug_assertions, target_os = "linux") },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -0,0 +1,102 @@
+use gdbstub::common::Signal;
+use gdbstub::conn::ConnectionExt;
+use gdbstub::stub::run_blocking::{self, WaitForStopReasonError};
+use gdbstub::stub::{BaseStopReason, DisconnectReason, GdbStub, SingleThreadStopReason};
+
+use super::x86_64_target::HyperlightSandboxTarget;
+use super::{DebugResponse, VcpuStopReason};
+
+pub struct GdbBlockingEventLoop;
+
+impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
+    type Connection = Box<dyn ConnectionExt<Error = std::io::Error>>;
+    type StopReason = SingleThreadStopReason<u64>;
+    type Target = HyperlightSandboxTarget;
+
+    fn wait_for_stop_reason(
+        target: &mut Self::Target,
+        conn: &mut Self::Connection,
+    ) -> Result<
+        run_blocking::Event<Self::StopReason>,
+        run_blocking::WaitForStopReasonError<
+            <Self::Target as gdbstub::target::Target>::Error,
+            <Self::Connection as gdbstub::conn::Connection>::Error,
+        >,
+    > {
+        loop {
+            match target.try_recv() {
+                Ok(DebugResponse::VcpuStopped(stop_reason)) => {
+                    log::debug!("VcpuStopped with reason {:?}", stop_reason);
+
+                    // Resume execution if unknown reason for stop
+                    let stop_response = match stop_reason {
+                        VcpuStopReason::DoneStep => BaseStopReason::DoneStep,
+                        VcpuStopReason::SwBp => BaseStopReason::SwBreak(()),
+                        VcpuStopReason::HwBp => BaseStopReason::HwBreak(()),
+                        VcpuStopReason::Unknown => {
+                            target
+                                .resume_vcpu()
+                                .map_err(WaitForStopReasonError::Target)?;
+
+                            continue;
+                        }
+                    };
+
+                    return Ok(run_blocking::Event::TargetStopped(stop_response));
+                }
+                Ok(msg) => {
+                    log::error!("Unexpected message received {:?}", msg);
+                }
+                Err(crossbeam_channel::TryRecvError::Empty) => (),
+                Err(crossbeam_channel::TryRecvError::Disconnected) => {
+                    return Ok(run_blocking::Event::TargetStopped(BaseStopReason::Exited(
+                        0,
+                    )));
+                }
+            }
+
+            if conn.peek().map(|b| b.is_some()).unwrap_or(false) {
+                let byte = conn
+                    .read()
+                    .map_err(run_blocking::WaitForStopReasonError::Connection)?;
+
+                return Ok(run_blocking::Event::IncomingData(byte));
+            }
+        }
+    }
+
+    fn on_interrupt(
+        _target: &mut Self::Target,
+    ) -> Result<Option<Self::StopReason>, <Self::Target as gdbstub::target::Target>::Error> {
+        Ok(Some(SingleThreadStopReason::SignalWithThread {
+            tid: (),
+            signal: Signal::SIGINT,
+        }))
+    }
+}
+
+pub fn event_loop_thread(
+    debugger: GdbStub<HyperlightSandboxTarget, Box<dyn ConnectionExt<Error = std::io::Error>>>,
+    target: &mut HyperlightSandboxTarget,
+) {
+    match debugger.run_blocking::<GdbBlockingEventLoop>(target) {
+        Ok(disconnect_reason) => match disconnect_reason {
+            DisconnectReason::Disconnect => {
+                log::info!("Gdb client disconnected");
+                if let Err(e) = target.disable_debug() {
+                    log::error!("Cannot disable debugging: {:?}", e);
+                }
+            }
+            DisconnectReason::TargetExited(_) => {
+                log::info!("Guest finalized execution and disconnected");
+            }
+            DisconnectReason::TargetTerminated(sig) => {
+                log::info!("Gdb target terminated with signal {}", sig)
+            }
+            DisconnectReason::Kill => log::info!("Gdb sent a kill command"),
+        },
+        Err(e) => {
+            log::error!("fatal error encountered: {e:?}");
+        }
+    }
+}