Skip to content

Commit 757fb9b

Browse files
dblnzdblnz
authored
Add support for Hyperlight KVM guest debugging using gdb (#111)
* add gdb cargo feature Signed-off-by: Doru Blânzeanu <[email protected]> * disable timeouts for messaging between host and hypervisor handler when gdb feature is on Signed-off-by: Doru Blânzeanu <[email protected]> * add guest debug port to sandbox configuration Signed-off-by: Doru Blânzeanu <[email protected]> * add gdb debug thread creation method Signed-off-by: Doru Blânzeanu <[email protected]> * add debug communication channel type - this type will be used by the gdb and the hypervisor handler to send requests and receive responses Signed-off-by: Doru Blânzeanu <[email protected]> * add hyperlight sandbox target to handle gdb commands support - the target implements the traits to provide callbacks for the gdb commands Signed-off-by: Doru Blânzeanu <[email protected]> * handle debug vcpu exit on kvm side - adds a specific handler for the vcpu exit debug that waits for debug messages and processes them Signed-off-by: Doru Blânzeanu <[email protected]> * add kvm debug configuration that handles vCPU interaction Signed-off-by: Doru Blânzeanu <[email protected]> * add read/write registers support Signed-off-by: Doru Blânzeanu <[email protected]> * hw breakpoint and resume support added Signed-off-by: Doru Blânzeanu <[email protected]> * add stepping support - also adds handling of gdb client disconnect by resuming execution Signed-off-by: Doru Blânzeanu <[email protected]> * add a way to read/write host shared mem when debugging Signed-off-by: Doru Blânzeanu <[email protected]> * add read/write address and text section offset support Signed-off-by: Doru Blânzeanu <[email protected]> * add sw breakpoint support Signed-off-by: Doru Blânzeanu <[email protected]> * add ci checks for the gdb feature Signed-off-by: Doru Blânzeanu <[email protected]> * add documentation about the gdb debugging feature Signed-off-by: Doru Blânzeanu <[email protected]> * add a way to report a non-fatal error to gdb - sometimes gdb tries to read from an address where the vcpu translate method fails, so we need to handle that in such a way that the hypervisor handler doesn't crash Signed-off-by: Doru Blânzeanu <[email protected]> * add CI gdb test Signed-off-by: Doru Blânzeanu <[email protected]> * add support for interrupts Signed-off-by: Doru Blânzeanu <[email protected]> * move gdb thread creation in hypervisor handler - this helps with separation and testing Signed-off-by: Doru Blânzeanu <[email protected]> * improve error handling and avoid panicking - replace `expect` statements with logic to propagate error up the call chain - improve log messages and comments Signed-off-by: Doru Blânzeanu <[email protected]> --------- Signed-off-by: Doru Blânzeanu <[email protected]>
1 parent 4b3bd98 commit 757fb9b

25 files changed

+2208
-10
lines changed

‎.github/workflows/dep_rust.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ jobs:
9595
# make sure certain cargo features compile
9696
cargo check -p hyperlight-host --features crashdump
9797
cargo check -p hyperlight-host --features print_debug
98+
cargo check -p hyperlight-host --features gdb
9899
99100
# without any driver (shouldn't compile)
100101
just test-rust-feature-compilation-fail ${{ matrix.config }}
@@ -114,6 +115,13 @@ jobs:
114115
RUST_LOG: debug
115116
run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
116117

118+
- name: Run Rust Gdb tests - linux
119+
if: runner.os == 'Linux' && matrix.hypervisor == 'kvm'
120+
env:
121+
CARGO_TERM_COLOR: always
122+
RUST_LOG: debug
123+
run: just test-rust-gdb-debugging ${{ matrix.config }}
124+
117125
### Benchmarks ###
118126
- name: Install github-cli (Linux mariner)
119127
if: runner.os == 'Linux' && matrix.hypervisor == 'mshv'

‎Cargo.lock

Lines changed: 32 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Justfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,11 @@ test-rust-feature-compilation-fail target=default-target:
110110
@# the following should fail on linux because one of kvm, mshv, or mshv3 feature must be specified, which is why the exit code is inverted with an !.
111111
{{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }}
112112

113+
# Test rust gdb debugging
114+
test-rust-gdb-debugging target=default-target: (build-rust target)
115+
{{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --example guest-debugging --features gdb
116+
{{ set-trace-env-vars }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --features gdb -- test_gdb
117+
113118
test target=default-target: (test-rust target)
114119

115120
# RUST LINTING

‎docs/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ This project is composed internally of several internal components, depicted in
3434

3535
* [Security guidance for developers](./security-guidance-for-developers.md)
3636
* [Paging Development Notes](./paging-development-notes.md)
37+
* [How to debug a Hyperlight guest](./how-to-debug-a-hyperlight-guest.md)
3738
* [How to use Flatbuffers in Hyperlight](./how-to-use-flatbuffers.md)
3839
* [How to make a Hyperlight release](./how-to-make-releases.md)
3940
* [Getting Hyperlight Metrics, Logs, and Traces](./hyperlight-metrics-logs-and-traces.md)

‎docs/how-to-debug-a-hyperlight-guest.md

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
# How to debug a Hyperlight **KVM** guest using gdb
2+
3+
Hyperlight supports gdb debugging of a **KVM** guest running inside a Hyperlight sandbox.
4+
When Hyperlight is compiled with the `gdb` feature enabled, a Hyperlight KVM sandbox can be configured
5+
to start listening for a gdb connection.
6+
7+
## Supported features
8+
9+
The Hyperlight `gdb` feature enables **KVM** guest debugging:
10+
- an entry point breakpoint is automatically set for the guest to stop
11+
- add and remove HW breakpoints (maximum 4 set breakpoints at a time)
12+
- add and remove SW breakpoints
13+
- read and write registers
14+
- read and write addresses
15+
- step/continue
16+
- get code offset from target
17+
18+
## Expected behavior
19+
20+
Below is a list describing some cases of expected behavior from a gdb debug
21+
session of a guest binary running inside a KVM Hyperlight sandbox.
22+
23+
- when the `gdb` feature is enabled and a SandboxConfiguration is provided a
24+
debug port, the created sandbox will wait for a gdb client to connect on the
25+
configured port
26+
- when the gdb client attaches, the guest vCPU is expected to be stopped at the
27+
entry point
28+
- if a gdb client disconnects unexpectedly, the debug session will be closed and
29+
the guest will continue executing disregarding any prior breakpoints
30+
- if multiple sandbox instances are created, each instance will have its own
31+
gdb thread listening on the configured port
32+
- if two sandbox instances are created with the same debug port, the second
33+
instance logs an error and the gdb thread will not be created, but the sandbox
34+
will continue to run without gdb debugging
35+
36+
## Example
37+
38+
### Sandbox configuration
39+
40+
The `guest-debugging` example in Hyperlight demonstrates how to configure a Hyperlight
41+
sandbox to listen for a gdb client on a specific port.
42+
43+
### CLI Gdb configuration
44+
45+
One can use a gdb config file to provide the symbols and desired configuration.
46+
47+
The below contents of the `.gdbinit` file can be used to provide a basic configuration
48+
to gdb startup.
49+
50+
```gdb
51+
# Path to symbols
52+
file path/to/symbols.elf
53+
# The port on which Hyperlight listens for a connection
54+
target remote :8080
55+
set disassembly-flavor intel
56+
set disassemble-next-line on
57+
enable pretty-printer
58+
layout src
59+
```
60+
One can find more information about the `.gdbinit` file at [gdbinit(5)](https://www.man7.org/linux/man-pages/man5/gdbinit.5.html).
61+
62+
### End to end example
63+
64+
Using the example mentioned at [Sandbox configuration](#sandbox-configuration)
65+
one can run the below commands to debug the guest binary:
66+
67+
```bash
68+
# Terminal 1
69+
$ cargo run --example guest-debugging --features gdb
70+
```
71+
72+
```bash
73+
# Terminal 2
74+
$ cat .gdbinit
75+
file src/tests/rust_guests/bin/debug/simpleguest
76+
target remote :8080
77+
set disassembly-flavor intel
78+
set disassemble-next-line on
79+
enable pretty-printer
80+
layout src
81+
82+
$ gdb
83+
```
84+
85+
### Using VSCode to debug a Hyperlight guest
86+
87+
To replicate the above behavior using VSCode follow the below steps:
88+
- install the `gdb` package on the host machine
89+
- install the `C/C++` extension in VSCode to add debugging capabilities
90+
- create a `.vscode/launch.json` file in the project directory with the below content:
91+
```json
92+
{
93+
"version": "0.2.0",
94+
"configurations": [
95+
{
96+
"name": "GDB",
97+
"type": "cppdbg",
98+
"request": "launch",
99+
"program": "${workspaceFolder}/src/tests/rust_guests/bin/debug/simpleguest",
100+
"args": [],
101+
"stopAtEntry": true,
102+
"hardwareBreakpoints": {"require": false, "limit": 4},
103+
"cwd": "${workspaceFolder}",
104+
"environment": [],
105+
"externalConsole": false,
106+
"MIMode": "gdb",
107+
"miDebuggerPath": "/usr/bin/gdb",
108+
"miDebuggerServerAddress": "localhost:8080",
109+
"setupCommands": [
110+
{
111+
"description": "Enable pretty-printing for gdb",
112+
"text": "-enable-pretty-printing",
113+
"ignoreFailures": true
114+
},
115+
{
116+
"description": "Set Disassembly Flavor to Intel",
117+
"text": "-gdb-set disassembly-flavor intel",
118+
"ignoreFailures": true
119+
}
120+
]
121+
}
122+
]
123+
}
124+
```
125+
- in `Run and Debug` tab, select the `GDB` configuration and click on the `Run`
126+
button to start the debugging session.
127+
The gdb client will connect to the Hyperlight sandbox and the guest vCPU will
128+
stop at the entry point.
129+
130+
131+
## How it works
132+
133+
The gdb feature is designed to work like a Request - Response protocol between
134+
a thread that accepts commands from a gdb client and the hypervisor handler over
135+
a communication channel.
136+
137+
All the functionality is implemented on the hypervisor side so it has access to
138+
the shared memory and the vCPU.
139+
140+
The gdb thread uses the `gdbstub` crate to handle the communication with the gdb client.
141+
When the gdb client requests one of the supported features mentioned above, a request
142+
is sent over the communication channel to the hypervisor handler for the sandbox
143+
to resolve.
144+
145+
Below is a sequence diagram that shows the interaction between the entities
146+
involved in the gdb debugging of a Hyperlight guest running inside a KVM sandbox.
147+
148+
```
149+
┌───────────────────────────────────────────────────────────────────────────────────────────────┐
150+
│ Hyperlight Sandbox │
151+
USER │ │
152+
┌────────────┐ │ ┌──────────────┐ ┌───────────────────────────┐ ┌────────┐ │
153+
│ gdb client │ │ │ gdb thread │ │ hypervisor handler thread │ │ vCPU │ │
154+
└────────────┘ │ └──────────────┘ └───────────────────────────┘ └────────┘ │
155+
| │ | create_gdb_thread | | │
156+
| │ |◄─────────────────────────────────────────┌─┐ vcpu stopped ┌─┐ │
157+
| attach │ ┌─┐ │ │◄──────────────────────────────┴─┘ │
158+
┌─┐───────────────────────┼────────►│ │ │ │ entrypoint breakpoint | │
159+
│ │ attach response │ │ │ │ │ | │
160+
│ │◄──────────────────────┼─────────│ │ │ │ | │
161+
│ │ │ │ │ │ │ | │
162+
│ │ add_breakpoint │ │ │ │ │ | │
163+
│ │───────────────────────┼────────►│ │ add_breakpoint │ │ | │
164+
│ │ │ │ │────────────────────────────────────────►│ │ add_breakpoint | │
165+
│ │ │ │ │ │ │────┐ | │
166+
│ │ │ │ │ │ │ │ | │
167+
│ │ │ │ │ │ │◄───┘ | │
168+
│ │ │ │ │ add_breakpoint response │ │ | │
169+
│ │ add_breakpoint response │ │◄────────────────────────────────────────│ │ | │
170+
│ │◄──────────────────────┬─────────│ │ │ │ | │
171+
│ │ continue │ │ │ │ │ | │
172+
│ │───────────────────────┼────────►│ │ continue │ │ | │
173+
│ │ │ │ │────────────────────────────────────────►│ │ resume vcpu | │
174+
│ │ │ │ │ │ │──────────────────────────────►┌─┐ │
175+
│ │ │ │ │ │ │ │ │ │
176+
│ │ │ │ │ │ │ │ │ │
177+
│ │ │ │ │ │ │ │ │ │
178+
│ │ │ │ │ │ │ │ │ │
179+
│ │ │ │ │ │ │ vcpu stopped │ │ │
180+
│ │ │ │ │ notify vcpu stop reason │ │◄──────────────────────────────┴─┘ │
181+
│ │ notify vcpu stop reason │ │◄────────────────────────────────────────│ │ | │
182+
│ │◄──────────────────────┬─────────│ │ │ │ | │
183+
│ │ continue until end │ │ │ │ │ | │
184+
│ │───────────────────────┼────────►│ │ continue │ │ resume vcpu | │
185+
│ │ │ │ │────────────────────────────────────────►│ │──────────────────────────────►┌─┐ │
186+
│ │ │ │ │ │ │ │ │ │
187+
│ │ │ │ │ comm channel disconnected │ │ vcpu halted │ │ │
188+
│ │ target finished exec│ │ │◄────────────────────────────────────────┤ │◄──────────────────────────────┴─┘ │
189+
│ │◄──────────────────────┼─────────┴─┘ target finished exec └─┘ | │
190+
│ │ │ | | | │
191+
└─┘ │ | | | │
192+
| └───────────────────────────────────────────────────────────────────────────────────────────────┘
193+
```

‎src/hyperlight_host/Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ sha256 = "1.4.0"
7171
windows-version = "0.1"
7272

7373
[target.'cfg(unix)'.dependencies]
74+
gdbstub = { version = "0.7.3", optional = true }
75+
gdbstub_arch = { version = "0.3.1", optional = true }
7476
seccompiler = { version = "0.4.0", optional = true }
7577
kvm-bindings = { version = "0.11", features = ["fam-wrappers"], optional = true }
7678
kvm-ioctls = { version = "0.20", optional = true }
@@ -128,6 +130,8 @@ kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
128130
mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
129131
mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
130132
inprocess = []
133+
# This enables easy debug in the guest
134+
gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
131135

132136
[[bench]]
133137
name = "benchmarks"

‎src/hyperlight_host/build.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ fn main() -> Result<()> {
8989
// Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
9090
// You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
9191
cfg_aliases::cfg_aliases! {
92+
gdb: { all(feature = "gdb", debug_assertions, feature = "kvm", target_os = "linux") },
9293
kvm: { all(feature = "kvm", target_os = "linux") },
9394
mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
9495
// inprocess feature is aliased with debug_assertions to make it only available in debug-builds.

0 commit comments

Comments
 (0)