Add support for Hyperlight KVM guest debugging using gdb

.github/workflows/dep_rust.yml

@@ -95,6 +95,7 @@ jobs:
           # make sure certain cargo features compile
           cargo check -p hyperlight-host --features crashdump
           cargo check -p hyperlight-host --features print_debug
+          cargo check -p hyperlight-host --features gdb
 
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}
@@ -114,6 +115,13 @@ jobs:
           RUST_LOG: debug
         run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
+      - name: Run Rust Gdb tests - linux
+        if: runner.os == 'Linux' && matrix.hypervisor == 'kvm'
+        env:
+          CARGO_TERM_COLOR: always
+          RUST_LOG: debug
+        run: just test-rust-gdb-debugging ${{ matrix.config }}
+
       ### Benchmarks ###
       - name: Install github-cli (Linux mariner)
         if: runner.os == 'Linux' && matrix.hypervisor == 'mshv'

Justfile

@@ -110,6 +110,11 @@ test-rust-feature-compilation-fail target=default-target:
     @# the following should fail on linux because one of kvm, mshv, or mshv3 feature must be specified, which is why the exit code is inverted with an !.
     {{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }}
 
+# Test rust gdb debugging
+test-rust-gdb-debugging target=default-target: (build-rust target)
+    {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging --features gdb
+    {{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --features gdb -- test_gdb
+
 test target=default-target: (test-rust target)
 
 # RUST LINTING

docs/README.md

@@ -34,6 +34,7 @@ This project is composed internally of several internal components, depicted in
 
 * [Security guidance for developers](./security-guidance-for-developers.md)
 * [Paging Development Notes](./paging-development-notes.md)
+* [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md)
 * [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md)
 * [How to make a Hyperlight release](./how-to-make-releases.md)
 * [Getting Hyperlight Metrics, Logs, and Traces](./hyperlight-metrics-logs-and-traces.md)

docs/how-to-debug-a-hyperlight-guest.md

@@ -0,0 +1,193 @@
+# How to debug a Hyperlight **KVM** guest using gdb
+
+Hyperlight supports gdb debugging of a **KVM** guest running inside a Hyperlight sandbox.
+When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight KVM sandbox can be configured
+to start listening for a gdb connection.
+
+## Supported features
+
+The Hyperlight `gdb` feature enables **KVM** guest debugging:
+   - an entry point breakpoint is automatically set for the guest to stop
+   - add and remove HW breakpoints (maximum 4 set breakpoints at a time)
+   - add and remove SW breakpoints
+   - read and write registers
+   - read and write addresses
+   - step/continue
+   - get code offset from target
+
+## Expected behavior
+
+Below is a list describing some cases of expected behavior from a gdb debug 
+session of a guest binary running inside a KVM Hyperlight sandbox.
+
+- when the `gdb` feature is enabled and a SandboxConfiguration is provided a
+  debug port, the created sandbox will wait for a gdb client to connect on the
+  configured port
+- when the gdb client attaches, the guest vCPU is expected to be stopped at the
+  entry point
+- if a gdb client disconnects unexpectedly, the debug session will be closed and
+  the guest will continue executing disregarding any prior breakpoints
+- if multiple sandbox instances are created, each instance will have its own
+  gdb thread listening on the configured port
+- if two sandbox instances are created with the same debug port, the second
+  instance logs an error and the gdb thread will not be created, but the sandbox
+  will continue to run without gdb debugging
+
+## Example
+
+### Sandbox configuration
+
+The `guest-debugging` example in Hyperlight demonstrates how to configure a Hyperlight
+sandbox to listen for a gdb client on a specific port.
+
+### CLI Gdb configuration
+
+One can use a gdb config file to provide the symbols and desired configuration.
+
+The below contents of the `.gdbinit` file can be used to provide a basic configuration
+to gdb startup.
+
+```gdb
+# Path to symbols
+file path/to/symbols.elf
+# The port on which Hyperlight listens for a connection
+target remote :8080
+set disassembly-flavor intel
+set disassemble-next-line on
+enable pretty-printer
+layout src
+```
+One can find more information about the `.gdbinit` file at [gdbinit(5)](https://www.man7.org/linux/man-pages/man5/gdbinit.5.html).
+
+### End to end example
+
+Using the example mentioned at [Sandbox configuration](#sandbox-configuration) 
+one can run the below commands to debug the guest binary:
+
+```bash
+# Terminal 1
+$ cargo run --example guest-debugging --features gdb
+```
+
+```bash
+# Terminal 2
+$ cat .gdbinit
+file src/tests/rust_guests/bin/debug/simpleguest
+target remote :8080
+set disassembly-flavor intel
+set disassemble-next-line on
+enable pretty-printer
+layout src
+
+$ gdb
+```
+
+### Using VSCode to debug a Hyperlight guest
+
+To replicate the above behavior using VSCode follow the below steps:
+- install the `gdb` package on the host machine
+- install the `C/C++` extension in VSCode to add debugging capabilities
+- create a `.vscode/launch.json` file in the project directory with the below content:
+    ```json
+    {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "GDB",
+                "type": "cppdbg",
+                "request": "launch",
+                "program": "${workspaceFolder}/src/tests/rust_guests/bin/debug/simpleguest",
+                "args": [],
+                "stopAtEntry": true,
+                "hardwareBreakpoints": {"require": false, "limit": 4},
+                "cwd": "${workspaceFolder}",
+                "environment": [],
+                "externalConsole": false,
+                "MIMode": "gdb",
+                "miDebuggerPath": "/usr/bin/gdb",
+                "miDebuggerServerAddress": "localhost:8080",
+                "setupCommands": [
+                    {
+                        "description": "Enable pretty-printing for gdb",
+                        "text": "-enable-pretty-printing",
+                        "ignoreFailures": true
+                    },
+                    {
+                        "description": "Set Disassembly Flavor to Intel",
+                        "text": "-gdb-set disassembly-flavor intel",
+                        "ignoreFailures": true
+                    }
+                ]
+            }
+        ]
+    }
+    ```
+- in `Run and Debug` tab, select the `GDB` configuration and click on the `Run`
+  button to start the debugging session.
+  The gdb client will connect to the Hyperlight sandbox and the guest vCPU will
+  stop at the entry point.
+
+
+## How it works
+
+The gdb feature is designed to work like a Request - Response protocol between
+a thread that accepts commands from a gdb client and the hypervisor handler over
+a communication channel.
+
+All the functionality is implemented on the hypervisor side so it has access to
+the shared memory and the vCPU.
+
+The gdb thread uses the `gdbstub` crate to handle the communication with the gdb client.
+When the gdb client requests one of the supported features mentioned above, a request
+is sent over the communication channel to the hypervisor handler for the sandbox
+to resolve.
+
+Below is a sequence diagram that shows the interaction between the entities
+involved in the gdb debugging of a Hyperlight guest running inside a KVM sandbox.
+
+```
+                               ┌───────────────────────────────────────────────────────────────────────────────────────────────┐
+                               │                                       Hyperlight Sandbox                                      │
+     USER                      │                                                                                               │
+┌────────────┐                 │  ┌──────────────┐                      ┌───────────────────────────┐              ┌────────┐  │
+│ gdb client │                 │  │  gdb thread  │                      │ hypervisor handler thread │              │  vCPU  │  │
+└────────────┘                 │  └──────────────┘                      └───────────────────────────┘              └────────┘  │
+      |                        │          |               create_gdb_thread           |                                 |      │
+      |                        │          |◄─────────────────────────────────────────┌─┐         vcpu stopped          ┌─┐     │
+      |    attach              │         ┌─┐                                         │ │◄──────────────────────────────┴─┘     │
+     ┌─┐───────────────────────┼────────►│ │                                         │ │     entrypoint breakpoint      |      │
+     │ │   attach response     │         │ │                                         │ │                                |      │
+     │ │◄──────────────────────┼─────────│ │                                         │ │                                |      │
+     │ │                       │         │ │                                         │ │                                |      │
+     │ │  add_breakpoint       │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │          add_breakpoint                 │ │                                |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │  add_breakpoint                |      │
+     │ │                       │         │ │                                         │ │────┐                           |      │
+     │ │                       │         │ │                                         │ │    │                           |      │
+     │ │                       │         │ │                                         │ │◄───┘                           |      │
+     │ │                       │         │ │          add_breakpoint response        │ │                                |      │
+     │ │  add_breakpoint response        │ │◄────────────────────────────────────────│ │                                |      │
+     │ │◄──────────────────────┬─────────│ │                                         │ │                                |      │
+     │ │   continue            │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │            continue                     │ │                                |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │         resume vcpu            |      │
+     │ │                       │         │ │                                         │ │──────────────────────────────►┌─┐     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │                                         │ │         vcpu stopped          │ │     │
+     │ │                       │         │ │        notify vcpu stop reason          │ │◄──────────────────────────────┴─┘     │
+     │ │   notify vcpu stop reason       │ │◄────────────────────────────────────────│ │                                |      │
+     │ │◄──────────────────────┬─────────│ │                                         │ │                                |      │
+     │ │    continue until end │         │ │                                         │ │                                |      │
+     │ │───────────────────────┼────────►│ │            continue                     │ │         resume vcpu            |      │
+     │ │                       │         │ │────────────────────────────────────────►│ │──────────────────────────────►┌─┐     │
+     │ │                       │         │ │                                         │ │                               │ │     │
+     │ │                       │         │ │        comm channel disconnected        │ │         vcpu halted           │ │     │
+     │ │   target finished exec│         │ │◄────────────────────────────────────────┤ │◄──────────────────────────────┴─┘     │
+     │ │◄──────────────────────┼─────────┴─┘          target finished exec           └─┘                                |      │
+     │ │                       │          |                                           |                                 |      │
+     └─┘                       │          |                                           |                                 |      │
+      |                        └───────────────────────────────────────────────────────────────────────────────────────────────┘
+```

src/hyperlight_host/Cargo.toml

@@ -71,6 +71,8 @@ sha256 = "1.4.0"
 windows-version = "0.1"
 
 [target.'cfg(unix)'.dependencies]
+gdbstub = { version = "0.7.3", optional = true }
+gdbstub_arch = { version = "0.3.1", optional = true }
 seccompiler = { version = "0.4.0", optional = true }
 kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true }
 kvm-ioctls = { version = "0.20", optional = true }
@@ -128,6 +130,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 inprocess = []
+# This enables easy debug in the guest
+gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
 
 [[bench]]
 name = "benchmarks"

src/hyperlight_host/build.rs

@@ -89,6 +89,7 @@ fn main() -> Result<()> {
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
+        gdb: { all(feature = "gdb", debug_assertions, feature = "kvm", target_os = "linux") },
         kvm: { all(feature = "kvm", target_os = "linux") },
         mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.