diff --git a/.gitignore b/.gitignore index 155e4cbd8a8..072b49ae674 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ test_results/* /resources/linux /resources/x86_64 /resources/aarch64 +llvm.sh +vmlinux +vmlinux.bin diff --git a/CHANGELOG.md b/CHANGELOG.md index ead297b7b84..7416664a80f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,29 @@ and this project adheres to ### Added +- [#5139](https://github.com/firecracker-microvm/firecracker/pull/5139): Added + support for [PVTime](https://docs.kernel.org/virt/kvm/arm/pvtime.html). This + is used to support steal time on ARM machines. +- [#5175](https://github.com/firecracker-microvm/firecracker/pull/5175): Allow + including a custom cpu template directly in the json configuration file passed + to `--config-file` under the `cpu_config` key. + +### Changed + +- [#5165](https://github.com/firecracker-microvm/firecracker/pull/5165): Changed + Firecracker snapshot feature from developer preview to generally available. + Incremental snapshots remain in developer preview. + +### Deprecated + +### Removed + +### Fixed + +## [1.12.0] + +### Added + - [#5048](https://github.com/firecracker-microvm/firecracker/pull/5048): Added support for [PVH boot mode](docs/pvh.md). This is used when an x86 kernel provides the appropriate ELF Note to indicate that PVH boot mode is supported. @@ -21,6 +44,10 @@ and this project adheres to so users need to regenerate snapshots. - [#4731](https://github.com/firecracker-microvm/firecracker/pull/4731): Added support for modifying the host TAP device name during snapshot restore. +- [#5146](https://github.com/firecracker-microvm/firecracker/pull/5146): Added + Intel Sapphire Rapids as a supported and tested platform for Firecracker. +- [#5148](https://github.com/firecracker-microvm/firecracker/pull/5148): Added + ARM Graviton4 as a supported and tested platform for Firecracker. ### Changed @@ -32,9 +59,6 @@ and this project adheres to Clarified what CPU models are supported by each existing CPU template. Firecracker exits with an error if a CPU template is used on an unsupported CPU model. -- [#5165](https://github.com/firecracker-microvm/firecracker/pull/5165): Changed - Firecracker snapshot feature from developer preview to generally available. - Incremental snapshots remain in developer preview. ### Deprecated @@ -45,12 +69,10 @@ and this project adheres to misnamed, as the value Firecracker sets it to is actually the page size in _bytes_, not KiB. It will be removed in Firecracker 2.0. -### Removed - ### Fixed -- #\[[5074](https://github.com/firecracker-microvm/firecracker/pull/5074)\] Fix - the `SendCtrlAltDel` command not working for ACPI-enabled guest kernels, by +- [#5074](https://github.com/firecracker-microvm/firecracker/pull/5074) Fix the + `SendCtrlAltDel` command not working for ACPI-enabled guest kernels, by dropping the i8042.nopnp argument from the default kernel command line Firecracker constructs. - [#5122](https://github.com/firecracker-microvm/firecracker/pull/5122): Keep diff --git a/Cargo.lock b/Cargo.lock index 39fd9998f4d..e35a1ac1bca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ dependencies = [ "displaydoc", "thiserror 2.0.12", "vm-memory", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -126,9 +126,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-fips-sys" -version = "0.13.5" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d9c2e952a1f57e8cbc78b058a968639e70c4ce8b9c0a5e6363d4e5670eed795" +checksum = "e99d74bb793a19f542ae870a6edafbc5ecf0bc0ba01d4636b7f7e0aba9ee9bd3" dependencies = [ "bindgen 0.69.5", "cc", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.1" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ddeb19ee86cb16ecfc871e5b0660aff6285760957aaedda6284cf0e790d3769" +checksum = "bfa9b6986f250236c27e5a204062434a773a13243d2ffc2955f37bdba4c5c6a1" dependencies = [ "bindgen 0.69.5", "cc", @@ -218,7 +218,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -268,9 +268,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.19" +version = "1.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" dependencies = [ "jobserver", "libc", @@ -419,7 +419,7 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "cpu-template-helper" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "clap", "displaydoc", @@ -607,7 +607,7 @@ dependencies = [ [[package]] name = "firecracker" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "cargo_toml", "displaydoc", @@ -670,9 +670,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", @@ -681,14 +681,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -719,9 +719,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" [[package]] name = "heck" @@ -731,9 +731,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "home" @@ -789,6 +789,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -806,7 +815,7 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jailer" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "libc", "log-instrument", @@ -818,9 +827,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ec30f7142be6fe14e1b021f50b85db8df2d4324ea6e91ec3e5dcde092021d0" +checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" dependencies = [ "jiff-static", "log", @@ -831,9 +840,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "526b834d727fd59d37b076b0c3236d9adde1b1729a4361e20b2026f738cc1dbe" +checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" dependencies = [ "proc-macro2", "quote", @@ -842,10 +851,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -966,7 +976,7 @@ dependencies = [ [[package]] name = "micro_http" version = "0.1.0" -source = "git+https://github.com/firecracker-microvm/micro-http#e854e50bc06a7e7bc0e5f5835d8f3f951e21f05f" +source = "git+https://github.com/firecracker-microvm/micro-http#4f621532e81ee2ad096a9c9592fdacc40d19de48" dependencies = [ "libc", "vmm-sys-util", @@ -1071,7 +1081,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -1118,6 +1128,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -1131,13 +1147,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.0", - "zerocopy 0.8.24", + "rand_core 0.9.3", ] [[package]] @@ -1157,7 +1172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1166,17 +1181,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] name = "rand_core" -version = "0.9.0" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", - "zerocopy 0.8.24", + "getrandom 0.3.2", ] [[package]] @@ -1190,7 +1204,7 @@ dependencies = [ [[package]] name = "rebase-snap" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "displaydoc", "libc", @@ -1265,7 +1279,7 @@ dependencies = [ [[package]] name = "seccompiler" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "bincode", "clap", @@ -1274,7 +1288,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.12", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -1344,7 +1358,7 @@ dependencies = [ [[package]] name = "snapshot-editor" -version = "1.12.0-dev" +version = "1.13.0-dev" dependencies = [ "clap", "clap-num", @@ -1372,9 +1386,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -1442,9 +1456,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.20" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ "serde", "serde_spanned", @@ -1454,26 +1468,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" + [[package]] name = "typenum" version = "1.18.0" @@ -1567,8 +1588,8 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", - "rand 0.9.0", + "getrandom 0.3.2", + "rand 0.9.1", "uuid-macro-internal", ] @@ -1684,7 +1705,7 @@ dependencies = [ "vm-memory", "vm-superio", "vmm-sys-util", - "zerocopy 0.8.24", + "zerocopy 0.8.25", ] [[package]] @@ -1717,9 +1738,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -1842,18 +1863,18 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "d9fb597c990f03753e08d3c29efbfcf2019a003b4bf4ba19225c158e1549f0f3" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags 2.9.0", ] @@ -1870,11 +1891,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive 0.8.25", ] [[package]] @@ -1890,9 +1911,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", diff --git a/docs/RELEASE_POLICY.md b/docs/RELEASE_POLICY.md index c9bd2a51089..f8f7006b721 100644 --- a/docs/RELEASE_POLICY.md +++ b/docs/RELEASE_POLICY.md @@ -90,8 +90,9 @@ v3.1 will be patched since were the last two Firecracker releases and less than | Release | Release Date | Latest Patch | Min. end of support | Official end of Support | | ------: | -----------: | -----------: | ------------------: | :------------------------------ | +| v1.12 | 2025-05-07 | v1.12.0 | 2025-11-07 | Supported | | v1.11 | 2025-03-18 | v1.11.0 | 2025-09-18 | Supported | -| v1.10 | 2024-11-07 | v1.10.1 | 2025-05-07 | Supported | +| v1.10 | 2024-11-07 | v1.10.1 | 2025-05-07 | 2025-05-07 (v1.12 released) | | v1.9 | 2024-09-02 | v1.9.1 | 2025-03-02 | 2025-03-18 (v1.11 released) | | v1.8 | 2024-07-10 | v1.8.0 | 2025-01-10 | 2025-01-10 (end of 6mo support) | | v1.7 | 2024-03-18 | v1.7.0 | 2024-09-18 | 2024-09-18 (end of 6mo support) | diff --git a/docs/pgo-getting-started.md b/docs/pgo-getting-started.md new file mode 100644 index 00000000000..322e2733748 --- /dev/null +++ b/docs/pgo-getting-started.md @@ -0,0 +1,213 @@ +# Profile-Guided Optimization (PGO) for Firecracker + +This document provides a guide for building Firecracker using Profile-Guided +Optimization (PGO) in an isolated manner. + +PGO can help improve performance by using runtime profiling data to guide +compiler optimizations. This process is fully **optional** and does **not** +affect the default Firecracker build system. + +## Overview + +PGO allows the Rust compiler (via LLVM) to use runtime profiling data to +optimize the generated binary for actual workloads. This generally results in +performance improvements for CPU-bound applications like Firecracker. + +We generate .profraw files at runtime and merge them into .profddata files to +then be reused in an optimized build of Firecracker. + +The PGO build process involves three main phases: + +1. **Instrumentation**: Build Firecracker with instrumentation to collect + profiling data. +1. **Profiling**: Run realistic workloads to generate `.profraw` profiling + files. +1. **Optimize**: Rebuild Firecracker using the collected profiling data for + improved performance. + +## 1. Build with Instrumentation + +Build Firecracker with profiling instrumentation enabled. If starting in the +`firecracker` directory, the command will be: + +``` +./tools/devtool pgo_build instrument +``` + +This produces a binary that, when executed, generates `.profraw` files +containing runtime behavior data. + +______________________________________________________________________ + +\*\* Note: the ideal environment for PGO is the same as the ideal environment +for firecracker: x86_64 architecture, Ubuntu OS (24.04 currently), and bare +metal (so that /dev/kvm is exposed). However, this step specifically can be done +on non-x86_64 machines with +`RUSTFLAGS="-Cprofile-generate=/tmp/firecracker-profdata" cargo build --release --package firecracker`. + +### Common Issue: Failed to run custom build command for `cpu-template-helper` + +Try: Ensuring the build directory exists and is writable with: + +``` +mkdir -p src/cpu-template-helper/build +chmod -R u+rw src/cpu-template-helper/build +``` + +Also ensure all dependencies (e.g., aws-lc-sys, userfaultfd-sys) can be built by +running: + +``` +cargo clean +cargo build --release +``` + +### Common Issue: failed to run custom build command for userfaultfd-sys v0.5.0 + +Try: `sudo apt install libclang-dev clang pkg-config` + +### Common Issue: failed to run custom build command for aws-lc-sys v0.28.1 + +Try: `sudo apt install cmake ninja-build perl` + +### Common Issue: a bunch of errors like.. + +``` +OUTPUT: Failed to compile memcmp_invalid_stripped_check +note: run with RUST_BACKTRACE=1 environment variable to display a backtrace +warning: build failed, waiting for other jobs to finish... +``` + +You might have an issue with global include-path overrides. + +## 2. Profiling + +Run realistic workloads to generate these `.profraw` files. Here are some +examples of typical workloads: + +- Boot a microVM +- Simulate network activity on a microVM +- Simulate basic I/O on a microVM + +Try to touch all major systems you personally care about optimizing so that you +can benchmark it against the base build later. + +Here's an example process of booting a minimal microVM: + +1. Download a test kernel and rootfs. +1. Start Firecracker +1. Use curl to configure in another terminal. E.g., + +``` +# Configure boot source +curl --unix-socket $API_SOCKET -i \ + -X PUT 'http://localhost/boot-source' \ + -H 'Content-Type: application/json' \ + -d '{ + "kernel_image_path": "vmlinux.bin", + "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + }' + +# Configure rootfs +curl --unix-socket $API_SOCKET -i \ + -X PUT 'http://localhost/drives/rootfs' \ + -H 'Content-Type: application/json' \ + -d '{ + "drive_id": "rootfs", + "path_on_host": "rootfs.ext4", + "is_root_device": true, + "is_read_only": false + }' + +# (Optional) set machine config if you want custom vCPU/RAM: +curl --unix-socket $API_SOCKET -i \ + -X PUT 'http://localhost/machine-config' \ + -H 'Content-Type: application/json' \ + -d '{ + "vcpu_count": 1, + "mem_size_mib": 128 + }' + +# Start the VM +curl --unix-socket $API_SOCKET -i \ + -X PUT 'http://localhost/actions' \ + -H 'Content-Type: application/json' \ + -d '{"action_type":"InstanceStart"}' +``` + +Please refer to the Firecracker getting started guide +[(link here)](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md) +for a more in-depth look at how to do this. + +## 3. Optimize + +After running your desired workloads, the resulting `.profraw` files can be seen +with: + +``` +ls /tmp/firecracker-profdata/ +``` + +______________________________________________________________________ + +#### Merging + +To merge these files into valid profile data use: + +``` +./tools/devtool pgo_build merge +``` + +#### Common Issue: version mismatch + +This will look something like: “raw profile format version = 10; expected +version = 9” + +This is common and might even happen on an ideal environment due to the Rust +toolchain producing v10 profile but Ubuntu 24.04 packages not shipping an +llvm-profdata that works for v10. You may be able to install the matching LLVM +on your host, but if it gives you trouble, using Rust's nightly toolchain can +also work. + +To use nightly, try: + +``` +rustup toolchain install nightly +rustup component add llvm-tools-preview --toolchain nightly + +export PATH="$HOME/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin:$PATH" +``` + +______________________________________________________________________ + +#### Optimized Build + +Once the `.profraw` files are merged into `.profdata`, you can re-build with the +merged profile: + +``` +./tools/devtool pgo_build optimize +``` + +Then, you can verify your optimized binary is in +`build/cargo_target/release/firecracker` and run it with + +``` +./build/cargo_target/release/firecracker --api-sock /tmp/fc.socket +``` + +### 4. Verify/Benchmark + +Once you have this PGO build, you can run any of the repository's existing +performance tests to observe the speed-ups. + +### Community Benchmark Results + +Please feel free to fork this repo, run your own benchmarks, and submit a PR +updating the table below. + +| Machine (CPU/RAM) | Firecracker (non-PGO) | Firecracker (PGO) | Δ (PGO vs. baseline) | Notes | +| ----------------------------- | --------------------: | ----------------: | -------------------: | -------------------------------------------- | +| AMD Ryzen 7 7700X; 32 GiB RAM | 0.01275 | 0.01079 | -15.37% | Ubuntu 24.04; used test_boottime.py for both | +| | | | | | +| | | | | | diff --git a/resources/rootfs.ext4 b/resources/rootfs.ext4 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/resources/vmlinux b/resources/vmlinux new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/acpi-tables/Cargo.toml b/src/acpi-tables/Cargo.toml index 141f09a47da..b31ebe4ac77 100644 --- a/src/acpi-tables/Cargo.toml +++ b/src/acpi-tables/Cargo.toml @@ -10,7 +10,7 @@ license = "Apache-2.0" displaydoc = "0.2.5" thiserror = "2.0.12" vm-memory = { version = "0.16.1", features = ["backend-mmap", "backend-bitmap"] } -zerocopy = { version = "0.8.24", features = ["derive"] } +zerocopy = { version = "0.8.25", features = ["derive"] } [lib] bench = false diff --git a/src/clippy-tracing/Cargo.toml b/src/clippy-tracing/Cargo.toml index d9883a03dc5..97f6a6c2275 100644 --- a/src/clippy-tracing/Cargo.toml +++ b/src/clippy-tracing/Cargo.toml @@ -14,7 +14,7 @@ clap = { version = "4.5.37", features = ["derive"] } itertools = "0.14.0" proc-macro2 = { version = "1.0.95", features = ["span-locations"] } quote = "1.0.40" -syn = { version = "2.0.100", features = ["full", "extra-traits", "visit", "visit-mut", "printing"] } +syn = { version = "2.0.101", features = ["full", "extra-traits", "visit", "visit-mut", "printing"] } walkdir = "2.5.0" [dev-dependencies] diff --git a/src/cpu-template-helper/Cargo.toml b/src/cpu-template-helper/Cargo.toml index 4993c957e59..8328c18a7c0 100644 --- a/src/cpu-template-helper/Cargo.toml +++ b/src/cpu-template-helper/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cpu-template-helper" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index 116c9e3ea2e..1a7cd3116b1 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecracker" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" build = "build.rs" diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index dd834baa785..61a0057a1ad 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 1.12.0-dev + version: 1.13.0-dev termsOfService: "" contact: email: "firecracker-maintainers@amazon.com" diff --git a/src/jailer/Cargo.toml b/src/jailer/Cargo.toml index 46893af00b5..9a25866a920 100644 --- a/src/jailer/Cargo.toml +++ b/src/jailer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jailer" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" description = "Process for starting Firecracker in production scenarios; applies a cgroup/namespace isolation barrier and then drops privileges." diff --git a/src/jailer/src/cgroup.rs b/src/jailer/src/cgroup.rs index 94098d3698f..8128a2b482f 100644 --- a/src/jailer/src/cgroup.rs +++ b/src/jailer/src/cgroup.rs @@ -504,13 +504,15 @@ pub mod test_util { use std::io::Write; use std::path::{Path, PathBuf}; - use vmm_sys_util::rand; + use vmm_sys_util::tempdir::TempDir; #[derive(Debug)] pub struct MockCgroupFs { mounts_file: File, - pub proc_mounts_path: String, - pub sys_cgroups_path: String, + // kept to clean up on Drop + _mock_jailer_dir: TempDir, + pub proc_mounts_path: PathBuf, + pub sys_cgroups_path: PathBuf, } // Helper object that simulates the layout of the cgroup file system @@ -533,17 +535,12 @@ pub mod test_util { } pub fn new() -> std::result::Result { - let mock_jailer_dir = format!( - "/tmp/firecracker/test/{}/jailer", - rand::rand_alphanumerics(4).into_string().unwrap() - ); - let mock_proc_mounts = format!("{}/{}", mock_jailer_dir, "proc/mounts",); - let mock_sys_cgroups = format!("{}/{}", mock_jailer_dir, "sys_cgroup",); - - let mock_proc_dir = Path::new(&mock_proc_mounts).parent().unwrap(); + let mock_jailer_dir = TempDir::new().unwrap(); + let mock_proc_mounts = mock_jailer_dir.as_path().join("proc/mounts"); + let mock_sys_cgroups = mock_jailer_dir.as_path().join("sys_cgroup"); // create a mock /proc/mounts file in a temporary directory - fs::create_dir_all(mock_proc_dir)?; + fs::create_dir_all(mock_proc_mounts.parent().unwrap())?; let file = OpenOptions::new() .read(true) .write(true) @@ -552,6 +549,7 @@ pub mod test_util { .open(mock_proc_mounts.clone())?; Ok(MockCgroupFs { mounts_file: file, + _mock_jailer_dir: mock_jailer_dir, proc_mounts_path: mock_proc_mounts, sys_cgroups_path: mock_sys_cgroups, }) @@ -563,9 +561,9 @@ pub mod test_util { writeln!( self.mounts_file, "cgroupv2 {}/unified cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate 0 0", - self.sys_cgroups_path, + self.sys_cgroups_path.to_str().unwrap(), )?; - let cg_unified_path = PathBuf::from(format!("{}/unified", self.sys_cgroups_path)); + let cg_unified_path = self.sys_cgroups_path.join("unified"); fs::create_dir_all(&cg_unified_path)?; Self::create_file_with_contents( cg_unified_path.join("cgroup.controllers"), @@ -589,26 +587,14 @@ pub mod test_util { writeln!( self.mounts_file, "cgroup {}/{} cgroup rw,nosuid,nodev,noexec,relatime,{} 0 0", - self.sys_cgroups_path, c, c, + self.sys_cgroups_path.to_str().unwrap(), + c, + c, )?; } Ok(()) } } - - // Cleanup created files when object goes out of scope - impl Drop for MockCgroupFs { - fn drop(&mut self) { - let tmp_dir = Path::new(self.proc_mounts_path.as_str()) - .parent() - .unwrap() - .parent() - .unwrap() - .parent() - .unwrap(); - let _ = fs::remove_dir_all(tmp_dir); - } - } } #[cfg(test)] @@ -639,14 +625,16 @@ mod tests { #[test] fn test_cgroup_conf_builder_invalid_version() { let mock_cgroups = MockCgroupFs::new().unwrap(); - let builder = CgroupConfigurationBuilder::new(0, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(0, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap_err(); } #[test] fn test_cgroup_conf_builder_no_mounts() { let mock_cgroups = MockCgroupFs::new().unwrap(); - let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap_err(); } @@ -654,7 +642,8 @@ mod tests { fn test_cgroup_conf_builder_v1() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap(); } @@ -662,7 +651,8 @@ mod tests { fn test_cgroup_conf_builder_v2() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v2_mounts().unwrap(); - let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap(); } @@ -670,14 +660,16 @@ mod tests { fn test_cgroup_conf_builder_v2_with_v1_mounts() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap_err(); } #[test] fn test_cgroup_conf_builder_v2_no_mounts() { let mock_cgroups = MockCgroupFs::new().unwrap(); - let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap_err(); } @@ -685,7 +677,8 @@ mod tests { fn test_cgroup_conf_builder_v1_with_v2_mounts() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v2_mounts().unwrap(); - let builder = CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap_err(); } @@ -696,9 +689,11 @@ mod tests { mock_cgroups.add_v2_mounts().unwrap(); for v in &[1, 2] { - let mut builder = - CgroupConfigurationBuilder::new(*v, mock_cgroups.proc_mounts_path.as_str()) - .unwrap(); + let mut builder = CgroupConfigurationBuilder::new( + *v, + mock_cgroups.proc_mounts_path.to_str().unwrap(), + ) + .unwrap(); builder .add_cgroup_property( @@ -719,9 +714,11 @@ mod tests { mock_cgroups.add_v2_mounts().unwrap(); for v in &[1, 2] { - let mut builder = - CgroupConfigurationBuilder::new(*v, mock_cgroups.proc_mounts_path.as_str()) - .unwrap(); + let mut builder = CgroupConfigurationBuilder::new( + *v, + mock_cgroups.proc_mounts_path.to_str().unwrap(), + ) + .unwrap(); builder .add_cgroup_property( "invalid.cg".to_string(), @@ -739,7 +736,8 @@ mod tests { mock_cgroups.add_v1_mounts().unwrap(); let mut builder = - CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + CgroupConfigurationBuilder::new(1, mock_cgroups.proc_mounts_path.to_str().unwrap()) + .unwrap(); builder .add_cgroup_property( "cpuset.mems".to_string(), @@ -750,7 +748,7 @@ mod tests { .unwrap(); let cg_conf = builder.build(); - let cg_root = PathBuf::from(format!("{}/cpuset", mock_cgroups.sys_cgroups_path)); + let cg_root = mock_cgroups.sys_cgroups_path.join("cpuset"); // with real cgroups these files are created automatically // since the mock will not do it automatically, we create it here @@ -773,11 +771,13 @@ mod tests { fn test_cgroup_conf_v2_write_value() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v2_mounts().unwrap(); - let builder = CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()); + let builder = + CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap()); builder.unwrap(); let mut builder = - CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + CgroupConfigurationBuilder::new(2, mock_cgroups.proc_mounts_path.to_str().unwrap()) + .unwrap(); builder .add_cgroup_property( "cpuset.mems".to_string(), @@ -787,7 +787,7 @@ mod tests { ) .unwrap(); - let cg_root = PathBuf::from(format!("{}/unified", mock_cgroups.sys_cgroups_path)); + let cg_root = mock_cgroups.sys_cgroups_path.join("unified"); assert_eq!(builder.get_v2_hierarchy_path().unwrap(), &cg_root); diff --git a/src/jailer/src/env.rs b/src/jailer/src/env.rs index e337ea95f90..e2953584a32 100644 --- a/src/jailer/src/env.rs +++ b/src/jailer/src/env.rs @@ -854,7 +854,7 @@ mod tests { arg_vec } - fn create_env(mock_proc_mounts: &str) -> Env { + fn create_env(mock_proc_mounts: &Path) -> Env { // Create a standard environment. let arg_parser = build_arg_parser(); let mut args = arg_parser.arguments().clone(); @@ -862,7 +862,7 @@ mod tests { let pseudo_exec_file_path = get_pseudo_exec_file_path(); args.parse(&make_args(&ArgVals::new(pseudo_exec_file_path.as_str()))) .unwrap(); - Env::new(&args, 0, 0, mock_proc_mounts).unwrap() + Env::new(&args, 0, 0, mock_proc_mounts.to_str().unwrap()).unwrap() } #[test] @@ -876,7 +876,7 @@ mod tests { let mut args = arg_parser.arguments().clone(); args.parse(&make_args(&good_arg_vals)).unwrap(); // This should be fine. - let good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()) + let good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()) .expect("This new environment should be created successfully."); let mut chroot_dir = PathBuf::from(good_arg_vals.chroot_base); @@ -902,8 +902,9 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&another_good_arg_vals)).unwrap(); - let another_good_env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()) - .expect("This another new environment should be created successfully."); + let another_good_env = + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()) + .expect("This another new environment should be created successfully."); assert!(!another_good_env.daemonize); assert!(!another_good_env.new_pid_ns); @@ -920,7 +921,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_res_limit_arg_vals = ArgVals { resource_limits: vec!["zzz"], @@ -930,7 +931,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_res_limit_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_id_arg_vals = ArgVals { id: "/ad./sa12", @@ -940,7 +941,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_id_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let inexistent_exec_file_arg_vals = ArgVals { exec_file: "/this!/file!/should!/not!/exist!/", @@ -951,7 +952,7 @@ mod tests { args = arg_parser.arguments().clone(); args.parse(&make_args(&inexistent_exec_file_arg_vals)) .unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_uid_arg_vals = ArgVals { uid: "zzz", @@ -961,7 +962,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_uid_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_gid_arg_vals = ArgVals { gid: "zzz", @@ -971,7 +972,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_gid_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_parent_cg_vals = ArgVals { parent_cgroup: Some("/root"), @@ -981,7 +982,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_parent_cg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_controller_pt = ArgVals { cgroups: vec!["../file_name=1", "./root=1", "/home=1"], @@ -990,7 +991,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_controller_pt)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); let invalid_format = ArgVals { cgroups: vec!["./root/", "../root"], @@ -999,7 +1000,7 @@ mod tests { let arg_parser = build_arg_parser(); args = arg_parser.arguments().clone(); args.parse(&make_args(&invalid_format)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); // The chroot-base-dir param is not validated by Env::new, but rather in run, when we // actually attempt to create the folder structure (the same goes for netns). @@ -1065,7 +1066,7 @@ mod tests { fn test_setup_jailed_folder() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let env = create_env(mock_cgroups.proc_mounts_path.as_str()); + let env = create_env(&mock_cgroups.proc_mounts_path); // Error case: non UTF-8 paths. let bad_string_bytes: Vec = vec![0, 102, 111, 111, 0]; // A leading nul followed by 'f', 'o', 'o' @@ -1136,28 +1137,37 @@ mod tests { fn test_mknod_and_own_dev() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let env = create_env(mock_cgroups.proc_mounts_path.as_str()); + let env = create_env(&mock_cgroups.proc_mounts_path); + + let mock_dev_dir = TempDir::new().unwrap(); // Ensure device nodes are created with correct major/minor numbers and permissions. - let mut dev_infos: Vec<(&CStr, u32, u32)> = vec![ - (c"/dev/net/tun-test", DEV_NET_TUN_MAJOR, DEV_NET_TUN_MINOR), - (c"/dev/kvm-test", DEV_KVM_MAJOR, DEV_KVM_MINOR), + let mut dev_infos: Vec<(PathBuf, u32, u32)> = vec![ + ( + mock_dev_dir.as_path().join("net/tun-test"), + DEV_NET_TUN_MAJOR, + DEV_NET_TUN_MINOR, + ), + ( + mock_dev_dir.as_path().join("kvm-test"), + DEV_KVM_MAJOR, + DEV_KVM_MINOR, + ), ]; if let Some(uffd_dev_minor) = env.uffd_dev_minor { - dev_infos.push((c"/dev/userfaultfd-test", DEV_UFFD_MAJOR, uffd_dev_minor)); + dev_infos.push(( + mock_dev_dir.as_path().join("userfaultfd-test"), + DEV_UFFD_MAJOR, + uffd_dev_minor, + )); } for (dev, major, minor) in dev_infos { - // Checking this just to be super sure there's no file at `dev_str` path (though - // it shouldn't be as we deleted it at the end of the previous test run). - if Path::new(dev.to_str().unwrap()).exists() { - fs::remove_file(dev.to_str().unwrap()).unwrap(); - } - - ensure_mknod_and_own_dev(&env, dev, major, minor); - // Remove the device node. - fs::remove_file(dev.to_str().unwrap()).expect("Could not remove file."); + // Ensure the folder where we are creating the node exists + fs::create_dir_all(dev.parent().unwrap()).unwrap(); + let dev_path = dev.to_str().map(CString::new).unwrap().unwrap(); + ensure_mknod_and_own_dev(&env, &dev_path, major, minor); } } @@ -1165,7 +1175,7 @@ mod tests { fn test_userfaultfd_dev() { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let env = create_env(mock_cgroups.proc_mounts_path.as_str()); + let env = create_env(&mock_cgroups.proc_mounts_path); if !Path::new(DEV_UFFD_PATH.to_str().unwrap()).exists() { assert_eq!(env.uffd_dev_minor, None); @@ -1207,7 +1217,8 @@ mod tests { let exec_file_name = Path::new(&some_arg_vals.exec_file).file_name().unwrap(); fs::write(some_arg_vals.exec_file, "some_content").unwrap(); args.parse(&make_args(&some_arg_vals)).unwrap(); - let mut env = Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + let mut env = + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap(); // Create the required chroot dir hierarchy. fs::create_dir_all(env.chroot_dir()).expect("Could not create dir hierarchy."); @@ -1270,7 +1281,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); // Check empty string let mut args = arg_parser.arguments().clone(); @@ -1279,7 +1290,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); // Check valid file empty value let mut args = arg_parser.arguments().clone(); @@ -1288,7 +1299,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); // Check valid file no value let mut args = arg_parser.arguments().clone(); @@ -1297,7 +1308,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap_err(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap_err(); // Cases that should succeed @@ -1308,7 +1319,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap(); // Check valid case let mut args = arg_parser.arguments().clone(); @@ -1317,7 +1328,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap(); // Check file with multiple "." let mut args = arg_parser.arguments().clone(); @@ -1326,7 +1337,7 @@ mod tests { ..good_arg_vals.clone() }; args.parse(&make_args(&invalid_cgroup_arg_vals)).unwrap(); - Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.as_str()).unwrap(); + Env::new(&args, 0, 0, mock_cgroups.proc_mounts_path.to_str().unwrap()).unwrap(); } #[test] @@ -1400,7 +1411,7 @@ mod tests { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let env = create_env(mock_cgroups.proc_mounts_path.as_str()); + let env = create_env(&mock_cgroups.proc_mounts_path); // Create the required chroot dir hierarchy. fs::create_dir_all(env.chroot_dir()).expect("Could not create dir hierarchy."); @@ -1427,7 +1438,7 @@ mod tests { let mut mock_cgroups = MockCgroupFs::new().unwrap(); mock_cgroups.add_v1_mounts().unwrap(); - let mut env = create_env(mock_cgroups.proc_mounts_path.as_str()); + let mut env = create_env(&mock_cgroups.proc_mounts_path); env.save_exec_file_pid(pid, PathBuf::from(exec_file_name)) .unwrap(); diff --git a/src/log-instrument-macros/Cargo.toml b/src/log-instrument-macros/Cargo.toml index 6f98f66721d..b41f836f0e8 100644 --- a/src/log-instrument-macros/Cargo.toml +++ b/src/log-instrument-macros/Cargo.toml @@ -13,7 +13,7 @@ bench = false [dependencies] proc-macro2 = "1.0.95" quote = "1.0.40" -syn = { version = "2.0.100", features = ["full", "extra-traits"] } +syn = { version = "2.0.101", features = ["full", "extra-traits"] } [lints] workspace = true diff --git a/src/rebase-snap/Cargo.toml b/src/rebase-snap/Cargo.toml index ece545b9003..1a6474e212e 100644 --- a/src/rebase-snap/Cargo.toml +++ b/src/rebase-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rebase-snap" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/seccompiler/Cargo.toml b/src/seccompiler/Cargo.toml index 152fe353ae2..b487e258ae2 100644 --- a/src/seccompiler/Cargo.toml +++ b/src/seccompiler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "seccompiler" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" description = "Program that compiles multi-threaded seccomp-bpf filters expressed as JSON into raw BPF programs, serializing them and outputting them to a file." @@ -23,7 +23,7 @@ libc = "0.2.172" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" thiserror = "2.0.12" -zerocopy = { version = "0.8.24" } +zerocopy = { version = "0.8.25" } [lints] workspace = true diff --git a/src/snapshot-editor/Cargo.toml b/src/snapshot-editor/Cargo.toml index 489cbf46503..59967281716 100644 --- a/src/snapshot-editor/Cargo.toml +++ b/src/snapshot-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snapshot-editor" -version = "1.12.0-dev" +version = "1.13.0-dev" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 39747ef1d53..86ba95e7768 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -43,7 +43,7 @@ vm-allocator = "0.1.0" vm-memory = { version = "0.16.1", features = ["backend-mmap", "backend-bitmap"] } vm-superio = "0.8.0" vmm-sys-util = { version = "0.12.1", features = ["with-serde"] } -zerocopy = { version = "0.8.24" } +zerocopy = { version = "0.8.25" } [target.'cfg(target_arch = "aarch64")'.dependencies] vm-fdt = "0.3.0" diff --git a/src/vmm/src/arch/aarch64/vcpu.rs b/src/vmm/src/arch/aarch64/vcpu.rs index 2c4c55375ed..59c00c3ff86 100644 --- a/src/vmm/src/arch/aarch64/vcpu.rs +++ b/src/vmm/src/arch/aarch64/vcpu.rs @@ -11,6 +11,7 @@ use std::mem::offset_of; use kvm_bindings::*; use kvm_ioctls::{VcpuExit, VcpuFd, VmFd}; use serde::{Deserialize, Serialize}; +use vm_memory::GuestAddress; use super::get_fdt_addr; use super::regs::*; @@ -42,6 +43,8 @@ pub enum VcpuArchError { Fam(vmm_sys_util::fam::Error), /// {0} GetMidrEl1(String), + /// Failed to set/get device attributes for vCPU: {0} + DeviceAttribute(kvm_ioctls::Error), } /// Extract the Manufacturer ID from the host. @@ -115,6 +118,8 @@ pub struct KvmVcpu { /// Vcpu peripherals, such as buses pub peripherals: Peripherals, kvi: kvm_vcpu_init, + /// IPA of steal_time region + pub pvtime_ipa: Option, } /// Vcpu peripherals @@ -148,6 +153,7 @@ impl KvmVcpu { fd: kvm_vcpu, peripherals: Default::default(), kvi, + pvtime_ipa: None, }) } @@ -243,6 +249,8 @@ impl KvmVcpu { // the boot state and turned secondary vcpus on. state.kvi.features[0] &= !(1 << KVM_ARM_VCPU_POWER_OFF); + state.pvtime_ipa = self.pvtime_ipa.map(|guest_addr| guest_addr.0); + Ok(state) } @@ -276,6 +284,13 @@ impl KvmVcpu { } self.set_mpstate(state.mp_state) .map_err(KvmVcpuError::RestoreState)?; + + // Assumes that steal time memory region was set up already + if let Some(pvtime_ipa) = state.pvtime_ipa { + self.enable_pvtime(GuestAddress(pvtime_ipa)) + .map_err(KvmVcpuError::RestoreState)?; + } + Ok(()) } @@ -439,6 +454,38 @@ impl KvmVcpu { pub fn set_mpstate(&self, state: kvm_mp_state) -> Result<(), VcpuArchError> { self.fd.set_mp_state(state).map_err(VcpuArchError::SetMp) } + + /// Check if pvtime (steal time on ARM) is supported for vcpu + pub fn supports_pvtime(&self) -> bool { + let pvtime_device_attr = kvm_bindings::kvm_device_attr { + group: kvm_bindings::KVM_ARM_VCPU_PVTIME_CTRL, + attr: kvm_bindings::KVM_ARM_VCPU_PVTIME_IPA as u64, + addr: 0, + flags: 0, + }; + + // Use kvm_has_device_attr to check if PVTime is supported + self.fd.has_device_attr(&pvtime_device_attr).is_ok() + } + + /// Enables pvtime for vcpu + pub fn enable_pvtime(&mut self, ipa: GuestAddress) -> Result<(), VcpuArchError> { + self.pvtime_ipa = Some(ipa); + + // Use KVM syscall (kvm_set_device_attr) to register the vCPU with the steal_time region + let vcpu_device_attr = kvm_bindings::kvm_device_attr { + group: KVM_ARM_VCPU_PVTIME_CTRL, + attr: KVM_ARM_VCPU_PVTIME_IPA as u64, + addr: &ipa.0 as *const u64 as u64, // userspace address of attr data + flags: 0, + }; + + self.fd + .set_device_attr(&vcpu_device_attr) + .map_err(VcpuArchError::DeviceAttribute)?; + + Ok(()) + } } impl Peripherals { @@ -467,6 +514,8 @@ pub struct VcpuState { pub mpidr: u64, /// kvi states for vcpu initialization. pub kvi: kvm_vcpu_init, + /// ipa for steal_time region + pub pvtime_ipa: Option, } impl Debug for VcpuState { diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 398c25ba056..4a810ee083a 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -15,6 +15,8 @@ use linux_loader::cmdline::Cmdline as LoaderKernelCmdline; use userfaultfd::Uffd; use utils::time::TimestampUs; #[cfg(target_arch = "aarch64")] +use vm_memory::GuestAddress; +#[cfg(target_arch = "aarch64")] use vm_superio::Rtc; use vm_superio::Serial; use vmm_sys_util::eventfd::EventFd; @@ -82,6 +84,9 @@ pub enum StartMicrovmError { CreateLegacyDevice(device_manager::legacy::LegacyDeviceError), /// Error creating VMGenID device: {0} CreateVMGenID(VmGenIdError), + /// Error enabling pvtime on vcpu: {0} + #[cfg(target_arch = "aarch64")] + EnablePVTime(crate::arch::VcpuArchError), /// Invalid Memory Configuration: {0} GuestMemory(crate::vstate::memory::MemoryError), /// Error with initrd initialization: {0}. @@ -289,6 +294,13 @@ pub fn build_microvm_for_boot( attach_vmgenid_device(&mut vmm)?; + #[cfg(target_arch = "aarch64")] + if vcpus[0].kvm_vcpu.supports_pvtime() { + setup_pvtime(&mut vmm, &mut vcpus)?; + } else { + log::warn!("Vcpus do not support pvtime, steal time will not be reported to guest"); + } + configure_system_for_boot( &mut vmm, vcpus.as_mut(), @@ -449,6 +461,16 @@ pub fn build_microvm_from_snapshot( } } + // Restore allocator state + #[cfg(target_arch = "aarch64")] + if let Some(pvtime_ipa) = vcpus[0].kvm_vcpu.pvtime_ipa { + allocate_pvtime_region( + &mut vmm, + vcpus.len(), + vm_allocator::AllocPolicy::ExactMatch(pvtime_ipa.0), + )?; + } + // Restore vcpus kvm state. for (vcpu, state) in vcpus.iter_mut().zip(microvm_state.vcpu_states.iter()) { vcpu.kvm_vcpu @@ -552,6 +574,44 @@ pub fn setup_serial_device( Ok(serial) } +/// 64 bytes due to alignment requirement in 3.1 of https://www.kernel.org/doc/html/v5.8/virt/kvm/devices/vcpu.html#attribute-kvm-arm-vcpu-pvtime-ipa +#[cfg(target_arch = "aarch64")] +const STEALTIME_STRUCT_MEM_SIZE: u64 = 64; + +/// Helper method to allocate steal time region +#[cfg(target_arch = "aarch64")] +fn allocate_pvtime_region( + vmm: &mut Vmm, + vcpu_count: usize, + policy: vm_allocator::AllocPolicy, +) -> Result { + let size = STEALTIME_STRUCT_MEM_SIZE * vcpu_count as u64; + let addr = vmm + .resource_allocator + .allocate_system_memory(size, STEALTIME_STRUCT_MEM_SIZE, policy) + .map_err(StartMicrovmError::AllocateResources)?; + Ok(GuestAddress(addr)) +} + +/// Sets up pvtime for all vcpus +#[cfg(target_arch = "aarch64")] +fn setup_pvtime(vmm: &mut Vmm, vcpus: &mut [Vcpu]) -> Result<(), StartMicrovmError> { + // Alloc sys mem for steal time region + let pvtime_mem: GuestAddress = + allocate_pvtime_region(vmm, vcpus.len(), vm_allocator::AllocPolicy::LastMatch)?; + + // Register all vcpus with pvtime device + for (i, vcpu) in vcpus.iter_mut().enumerate() { + vcpu.kvm_vcpu + .enable_pvtime(GuestAddress( + pvtime_mem.0 + i as u64 * STEALTIME_STRUCT_MEM_SIZE, + )) + .map_err(StartMicrovmError::EnablePVTime)?; + } + + Ok(()) +} + #[cfg(target_arch = "aarch64")] fn attach_legacy_devices_aarch64( event_manager: &mut EventManager, diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index aeacadeb66e..4111d8d6c34 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -148,7 +148,7 @@ pub enum CreateSnapshotError { } /// Snapshot version -pub const SNAPSHOT_VERSION: Version = Version::new(6, 0, 0); +pub const SNAPSHOT_VERSION: Version = Version::new(7, 0, 0); /// Creates a Microvm snapshot. pub fn create_snapshot( diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index 097e2041b55..70c317bb1e1 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -63,6 +63,13 @@ pub enum ResourcesError { EntropyDevice(#[from] EntropyDeviceError), } +#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] +#[serde(untagged)] +enum CustomCpuTemplateOrPath { + Path(PathBuf), + Template(CustomCpuTemplate), +} + /// Used for configuring a vmm from one single json passed to the Firecracker process. #[derive(Debug, Default, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] @@ -70,7 +77,7 @@ pub struct VmmConfig { balloon: Option, drives: Vec, boot_source: BootSourceConfig, - cpu_config: Option, + cpu_config: Option, logger: Option, machine_config: Option, metrics: Option, @@ -136,11 +143,18 @@ impl VmResources { resources.update_machine_config(&machine_config)?; } - if let Some(cpu_config) = vmm_config.cpu_config { - let cpu_config_json = - std::fs::read_to_string(cpu_config).map_err(ResourcesError::File)?; - let cpu_template = CustomCpuTemplate::try_from(cpu_config_json.as_str())?; - resources.set_custom_cpu_template(cpu_template); + if let Some(either) = vmm_config.cpu_config { + match either { + CustomCpuTemplateOrPath::Path(path) => { + let cpu_config_json = + std::fs::read_to_string(path).map_err(ResourcesError::File)?; + let cpu_template = CustomCpuTemplate::try_from(cpu_config_json.as_str())?; + resources.set_custom_cpu_template(cpu_template); + } + CustomCpuTemplateOrPath::Template(template) => { + resources.set_custom_cpu_template(template) + } + } } resources.build_boot_source(vmm_config.boot_source)?; @@ -505,6 +519,7 @@ mod tests { use super::*; use crate::HTTP_MAX_PAYLOAD_SIZE; + use crate::cpu_config::templates::test_utils::TEST_TEMPLATE_JSON; use crate::cpu_config::templates::{CpuTemplateType, StaticCpuTemplate}; use crate::devices::virtio::balloon::Balloon; use crate::devices::virtio::block::virtio::VirtioBlockError; @@ -1051,6 +1066,43 @@ mod tests { assert!(matches!(error, ResourcesError::File(_)), "{:?}", error); } + #[test] + fn test_cpu_config_inline() { + // Include custom cpu template directly inline in config json + let kernel_file = TempFile::new().unwrap(); + let rootfs_file = TempFile::new().unwrap(); + let default_instance_info = InstanceInfo::default(); + + let json = format!( + r#"{{ + "boot-source": {{ + "kernel_image_path": "{}", + "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" + }}, + "cpu-config": {}, + "drives": [ + {{ + "drive_id": "rootfs", + "path_on_host": "{}", + "is_root_device": true, + "is_read_only": false + }} + ] + }}"#, + kernel_file.as_path().to_str().unwrap(), + TEST_TEMPLATE_JSON, + rootfs_file.as_path().to_str().unwrap(), + ); + + VmResources::from_json( + json.as_str(), + &default_instance_info, + HTTP_MAX_PAYLOAD_SIZE, + None, + ) + .unwrap(); + } + #[test] fn test_cpu_config_from_valid_json() { // Valid cpu config file path. diff --git a/src/vmm/src/signal_handler.rs b/src/vmm/src/signal_handler.rs index 3b6162c7e4c..0fdfa0d8b97 100644 --- a/src/vmm/src/signal_handler.rs +++ b/src/vmm/src/signal_handler.rs @@ -50,7 +50,6 @@ macro_rules! generate_handler { $body(si_code, info); - #[cfg(not(test))] match si_signo { $signal_name => exit_with_code(crate::FcExitCode::$exit_code), _ => exit_with_code(FcExitCode::UnexpectedError), @@ -170,74 +169,3 @@ pub fn register_signal_handlers() -> vmm_sys_util::errno::Result<()> { register_signal_handler(SIGILL, sigill_handler)?; Ok(()) } - -#[cfg(test)] -mod tests { - #![allow(clippy::undocumented_unsafe_blocks)] - use std::{process, thread}; - - use libc::syscall; - - use super::*; - - #[test] - fn test_signal_handler() { - let child = thread::spawn(move || { - register_signal_handlers().unwrap(); - - // Call the forbidden `SYS_mkdirat`. - unsafe { libc::syscall(libc::SYS_mkdirat, "/foo/bar\0") }; - - // Call SIGBUS signal handler. - assert_eq!(METRICS.signals.sigbus.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGBUS); - } - - // Call SIGSEGV signal handler. - assert_eq!(METRICS.signals.sigsegv.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGSEGV); - } - - // Call SIGXFSZ signal handler. - assert_eq!(METRICS.signals.sigxfsz.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGXFSZ); - } - - // Call SIGXCPU signal handler. - assert_eq!(METRICS.signals.sigxcpu.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGXCPU); - } - - // Call SIGPIPE signal handler. - assert_eq!(METRICS.signals.sigpipe.count(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGPIPE); - } - - // Call SIGHUP signal handler. - assert_eq!(METRICS.signals.sighup.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGHUP); - } - - // Call SIGILL signal handler. - assert_eq!(METRICS.signals.sigill.fetch(), 0); - unsafe { - syscall(libc::SYS_kill, process::id(), SIGILL); - } - }); - child.join().unwrap(); - - assert!(METRICS.signals.sigbus.fetch() >= 1); - assert!(METRICS.signals.sigsegv.fetch() >= 1); - assert!(METRICS.signals.sigxfsz.fetch() >= 1); - assert!(METRICS.signals.sigxcpu.fetch() >= 1); - assert!(METRICS.signals.sigpipe.count() >= 1); - assert!(METRICS.signals.sighup.fetch() >= 1); - assert!(METRICS.signals.sigill.fetch() >= 1); - } -} diff --git a/tests/README.md b/tests/README.md index c306566392f..677544889ad 100644 --- a/tests/README.md +++ b/tests/README.md @@ -184,7 +184,7 @@ function to [`.buildkite/pipeline_perf.py`](../.buildkite/pipeline_perf.py). To manually run an A/B-Test, use ```sh -tools/devtool -y test --ab [optional arguments to ab_test.py] run --test +tools/devtool -y test --ab [optional arguments to ab_test.py] run --pytest-opts ``` Here, _dir A_ and _dir B_ are directories containing firecracker and jailer @@ -198,7 +198,7 @@ branch and the `HEAD` of your current branch, run ```sh tools/devtool -y build --rev main --release tools/devtool -y build --rev HEAD --release -tools/devtool -y test --no-build --ab -- run build/main build/HEAD --test integration_tests/performance/test_boottime.py::test_boottime +tools/devtool -y test --no-build --ab -- run build/main build/HEAD --pytest-opts integration_tests/performance/test_boottime.py::test_boottime ``` #### How to Write an A/B-Compatible Test and Common Pitfalls @@ -213,9 +213,9 @@ dimension to match up data series between two test runs. It only matches up two data series with the same name if their dimensions match. Special care needs to be taken when pytest expands the argument passed to -`tools/ab_test.py`'s `--test` option into multiple individual test cases. If two -test cases use the same dimensions for different data series, the script will -fail and print out the names of the violating data series. For this reason, +`tools/ab_test.py`'s `--pytest-opts` option into multiple individual test cases. +If two test cases use the same dimensions for different data series, the script +will fail and print out the names of the violating data series. For this reason, **A/B-Compatible tests should include a `performance_test` key in their dimension set whose value is set to the name of the test**. diff --git a/tests/integration_tests/functional/test_balloon.py b/tests/integration_tests/functional/test_balloon.py index dafa84b6451..2bfa82f4f1c 100644 --- a/tests/integration_tests/functional/test_balloon.py +++ b/tests/integration_tests/functional/test_balloon.py @@ -522,6 +522,8 @@ def test_balloon_snapshot(microvm_factory, guest_kernel, rootfs): # Get the stats after we take a snapshot and dirty some memory, # then reclaim it. + # Ensure we gave enough time for the stats to update. + time.sleep(STATS_POLLING_INTERVAL_S) latest_stats = microvm.api.balloon_stats.get().json() # Ensure the stats are still working after restore and show diff --git a/tests/integration_tests/functional/test_net.py b/tests/integration_tests/functional/test_net.py index 20a40e677b0..21fd11d924a 100644 --- a/tests/integration_tests/functional/test_net.py +++ b/tests/integration_tests/functional/test_net.py @@ -6,6 +6,7 @@ import time import pytest +from tenacity import Retrying, stop_after_attempt, wait_fixed from framework import utils @@ -118,5 +119,12 @@ def test_tap_offload(uvm_any): vm.netns.check_output(f"python3 ./host_tools/udp_offload.py {vm.ssh.host} {port}") # Check that the server received the message - ret = vm.ssh.run(f"sync ; cat {out_filename}") - assert ret.stdout == message, f"{ret.stdout=} {ret.stderr=}" + # Allow for some delay due to the asynchronous nature of the test + for attempt in Retrying( + stop=stop_after_attempt(10), + wait=wait_fixed(0.1), + reraise=True, + ): + with attempt: + ret = vm.ssh.check_output(f"sync; cat {out_filename}") + assert ret.stdout == message, f"{ret.stdout=} {ret.stderr=}" diff --git a/tests/integration_tests/functional/test_shut_down.py b/tests/integration_tests/functional/test_shut_down.py index 591f04e4593..4b21aa3d2d5 100644 --- a/tests/integration_tests/functional/test_shut_down.py +++ b/tests/integration_tests/functional/test_shut_down.py @@ -27,15 +27,6 @@ def test_reboot(uvm_plain_any): vm.add_net_iface() vm.start() - # Get Firecracker PID so we can count the number of threads. - firecracker_pid = vm.firecracker_pid - - # Get number of threads in Firecracker - cmd = "ps -o nlwp {} | tail -1 | awk '{{print $1}}'".format(firecracker_pid) - _, stdout, _ = utils.check_output(cmd) - nr_of_threads = stdout.rstrip() - assert int(nr_of_threads) == 6 - # Consume existing metrics lines = vm.get_all_metrics() assert len(lines) == 1 diff --git a/tests/integration_tests/functional/test_steal_time.py b/tests/integration_tests/functional/test_steal_time.py new file mode 100644 index 00000000000..d7c3c25cb92 --- /dev/null +++ b/tests/integration_tests/functional/test_steal_time.py @@ -0,0 +1,121 @@ +# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for verifying the PVTime device behavior under contention and across snapshots.""" + +import time + +import pytest + +from framework.properties import global_props + + +def get_steal_time_ms(vm): + """Returns total steal time of vCPUs in VM in milliseconds""" + _, out, _ = vm.ssh.run("grep -w '^cpu' /proc/stat") + steal_time_tck = int(out.strip().split()[8]) + clk_tck = int(vm.ssh.run("getconf CLK_TCK").stdout) + return steal_time_tck / clk_tck * 1000 + + +@pytest.mark.skipif( + global_props.cpu_architecture != "aarch64", reason="Only run in aarch64" +) +def test_guest_has_pvtime_enabled(uvm_plain): + """ + Check that the guest kernel has enabled PV steal time. + """ + vm = uvm_plain + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.start() + + _, stdout, _ = vm.ssh.run("dmesg | grep 'stolen time PV'") + assert ( + "stolen time PV" in stdout + ), "Guest kernel did not report PV steal time enabled" + + +def test_pvtime_steal_time_increases(uvm_plain): + """ + Test that PVTime steal time increases when both vCPUs are contended on the same pCPU. + """ + vm = uvm_plain + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.start() + + # Pin both vCPUs to the same physical CPU to induce contention + vm.pin_vcpu(0, 0) + vm.pin_vcpu(1, 0) + + # Start two infinite loops to hog CPU time + hog_cmd = "nohup bash -c 'while true; do :; done' >/dev/null 2>&1 &" + vm.ssh.run(hog_cmd) + vm.ssh.run(hog_cmd) + + # Measure before and after steal time + steal_before = get_steal_time_ms(vm) + time.sleep(2) + steal_after = get_steal_time_ms(vm) + + # Require increase in steal time + assert ( + steal_after > steal_before + ), f"Steal time did not increase as expected. Before: {steal_before}, After: {steal_after}" + + +def test_pvtime_snapshot(uvm_plain, microvm_factory): + """ + Test that PVTime steal time is preserved across snapshot/restore + and continues increasing post-resume. + """ + vm = uvm_plain + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.start() + + vm.pin_vcpu(0, 0) + vm.pin_vcpu(1, 0) + + hog_cmd = "nohup bash -c 'while true; do :; done' >/dev/null 2>&1 &" + vm.ssh.run(hog_cmd) + vm.ssh.run(hog_cmd) + + # Snapshot pre-steal time + steal_before = get_steal_time_ms(vm) + + snapshot = vm.snapshot_full() + vm.kill() + + # Restore microVM from snapshot and resume + restored_vm = microvm_factory.build() + restored_vm.spawn() + restored_vm.restore_from_snapshot(snapshot, resume=False) + snapshot.delete() + + restored_vm.pin_vcpu(0, 0) + restored_vm.pin_vcpu(1, 0) + restored_vm.resume() + + # Steal time just after restoring + steal_after_snap = get_steal_time_ms(restored_vm) + + time.sleep(2) + + # Steal time after running resumed VM + steal_after_resume = get_steal_time_ms(restored_vm) + + # Ensure steal time persisted and continued increasing + tolerance = 10000 # 10.0 seconds tolerance for persistence check + persisted = ( + steal_before < steal_after_snap and steal_after_snap - steal_before < tolerance + ) + increased = steal_after_resume > steal_after_snap + + assert ( + persisted and increased + ), "Steal time did not persist through snapshot or failed to increase after resume" diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py index 3bf74e3607a..5732a7e56f8 100644 --- a/tests/integration_tests/performance/test_boottime.py +++ b/tests/integration_tests/performance/test_boottime.py @@ -11,7 +11,6 @@ from framework.properties import global_props # Regex for obtaining boot time from some string. -TIMESTAMP_LOG_REGEX = r"Guest-boot-time\s+\=\s+(\d+)\s+us" DEFAULT_BOOT_ARGS = ( "reboot=k panic=1 pci=off nomodule 8250.nr_uarts=0" @@ -27,27 +26,32 @@ } -def _get_microvm_boottime(vm): +def get_boottime_device_info(vm): """Auxiliary function for asserting the expected boot time.""" boot_time_us = None + boot_time_cpu_us = None timestamps = [] + timestamp_log_regex = ( + r"Guest-boot-time =\s+(\d+) us\s+(\d+) ms,\s+(\d+) CPU us\s+(\d+) CPU ms" + ) + iterations = 50 sleep_time_s = 0.1 for _ in range(iterations): - timestamps = re.findall(TIMESTAMP_LOG_REGEX, vm.log_data) + timestamps = re.findall(timestamp_log_regex, vm.log_data) if timestamps: break time.sleep(sleep_time_s) if timestamps: - boot_time_us = int(timestamps[0]) + boot_time_us, _, boot_time_cpu_us, _ = timestamps[0] - assert boot_time_us, ( + assert boot_time_us and boot_time_cpu_us, ( f"MicroVM did not boot within {sleep_time_s * iterations}s\n" f"Firecracker logs:\n{vm.log_data}\n" f"Thread backtraces:\n{vm.thread_backtraces}" ) - return boot_time_us + return int(boot_time_us), int(boot_time_cpu_us) def find_events(log_data): @@ -69,6 +73,37 @@ def find_events(log_data): return timestamps +def get_systemd_analyze_times(microvm): + """ + Parse systemd-analyze output + """ + rc, stdout, stderr = microvm.ssh.run("systemd-analyze") + assert rc == 0, stderr + assert stderr == "" + + boot_line = stdout.splitlines()[0] + # The line will look like this: + # Startup finished in 79ms (kernel) + 231ms (userspace) = 310ms + # In the regex we capture the time and the unit for kernel, userspace and total values + pattern = r"Startup finished in (\d*)(ms|s)\s+\(kernel\) \+ (\d*)(ms|s)\s+\(userspace\) = ([\d.]*)(ms|s)\s*" + kernel, kernel_unit, userspace, userspace_unit, total, total_unit = re.findall( + pattern, boot_line + )[0] + + def to_ms(v, unit): + match unit: + case "ms": + return float(v) + case "s": + return float(v) * 1000 + + kernel = to_ms(kernel, kernel_unit) + userspace = to_ms(userspace, userspace_unit) + total = to_ms(total, total_unit) + + return kernel, userspace, total + + @pytest.mark.parametrize( "vcpu_count,mem_size_mib", [(1, 128), (1, 1024), (2, 2048), (4, 4096)], @@ -102,14 +137,28 @@ def test_boottime( vm.add_net_iface() vm.start() vm.pin_threads(0) - boottime_us = _get_microvm_boottime(vm) - metrics.put_metric("boot_time", boottime_us, unit="Microseconds") - timestamps = find_events(vm.log_data) - build_time = timestamps["build microvm for boot"]["duration"] - metrics.put_metric("build_time", build_time.microseconds, unit="Microseconds") + + boot_time_us, cpu_boot_time_us = get_boottime_device_info(vm) metrics.put_metric( "guest_boot_time", - boottime_us - build_time.microseconds, + boot_time_us, + unit="Microseconds", + ) + metrics.put_metric( + "guest_cpu_boot_time", + cpu_boot_time_us, unit="Microseconds", ) + + events = find_events(vm.log_data) + build_time = events["build microvm for boot"]["duration"] + metrics.put_metric("build_time", build_time.microseconds, unit="Microseconds") + resume_time = events["boot microvm"]["duration"] + metrics.put_metric("resume_time", resume_time.microseconds, unit="Microseconds") + + kernel, userspace, total = get_systemd_analyze_times(vm) + metrics.put_metric("systemd_kernel", kernel, unit="Milliseconds") + metrics.put_metric("systemd_userspace", userspace, unit="Milliseconds") + metrics.put_metric("systemd_total", total, unit="Milliseconds") + vm.kill() diff --git a/tools/ab_test.py b/tools/ab_test.py index 3790ff4674c..2ad3037e86d 100755 --- a/tools/ab_test.py +++ b/tools/ab_test.py @@ -49,6 +49,11 @@ # block latencies if guest uses async request submission {"fio_engine": "libaio", "metric": "clat_read"}, {"fio_engine": "libaio", "metric": "clat_write"}, + # boot time metrics + {"performance_test": "test_boottime", "metric": "resume_time"}, + # block throughput on m8g + {"fio_engine": "libaio", "vcpus": 2, "instance": "m8g.metal-24xl"}, + {"fio_engine": "libaio", "vcpus": 2, "instance": "m8g.metal-48xl"}, ] @@ -117,8 +122,6 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False): # Dictionary mapping EMF dimensions to A/B-testable metrics/properties processed_emf = {} - distinct_values_per_dimenson = defaultdict(set) - report = json.loads(report_path.read_text("UTF-8")) for test in report["tests"]: for line in test["teardown"]["stdout"].splitlines(): @@ -138,9 +141,6 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False): if not dimensions: continue - for dimension, value in dimensions.items(): - distinct_values_per_dimenson[dimension].add(value) - dimension_set = frozenset(dimensions.items()) if dimension_set not in processed_emf: @@ -157,24 +157,27 @@ def load_data_series(report_path: Path, tag=None, *, reemit: bool = False): values.extend(result[metric][0]) - irrelevant_dimensions = set() + return processed_emf - for dimension, distinct_values in distinct_values_per_dimenson.items(): - if len(distinct_values) == 1: - irrelevant_dimensions.add(dimension) - post_processed_emf = {} +def uninteresting_dimensions(processed_emf): + """ + Computes the set of cloudwatch dimensions that only ever take on a + single value across the entire dataset. + """ + values_per_dimension = defaultdict(set) + + for dimension_set in processed_emf: + for dimension, value in dimension_set: + values_per_dimension[dimension].add(value) - for dimension_set, metrics in processed_emf.items(): - processed_key = frozenset( - (dim, value) - for (dim, value) in dimension_set - if dim not in irrelevant_dimensions - ) + uninteresting = set() - post_processed_emf[processed_key] = metrics + for dimension, distinct_values in values_per_dimension.items(): + if len(distinct_values) == 1: + uninteresting.add(dimension) - return post_processed_emf + return uninteresting def collect_data(binary_dir: Path, pytest_opts: str): @@ -302,6 +305,7 @@ def analyze_data( ) messages = [] + do_not_print_list = uninteresting_dimensions(processed_emf_a) for dimension_set, metric, result, unit in failures: # Sanity check as described above if abs(statistics.mean(relative_changes_by_metric[metric])) <= noise_threshold: @@ -323,7 +327,7 @@ def analyze_data( f"for metric \033[1m{metric}\033[0m with \033[0;31m\033[1mp={result.pvalue}\033[0m. " f"This means that observing a change of this magnitude or worse, assuming that performance " f"characteristics did not change across the tested commits, has a probability of {result.pvalue:.2%}. " - f"Tested Dimensions:\n{json.dumps(dict(dimension_set), indent=2, sort_keys=True)}" + f"Tested Dimensions:\n{json.dumps(dict({k: v for k,v in dimension_set.items() if k not in do_not_print_list}), indent=2, sort_keys=True)}" ) messages.append(msg) diff --git a/tools/devtool b/tools/devtool index f86263c4731..8edc338a293 100755 --- a/tools/devtool +++ b/tools/devtool @@ -411,6 +411,9 @@ cmd_help() { echo " Downloads the CI artifacts used for testing from our S3 bucket. If --force is passed, purges any existing" echo " artifacts first. Useful for refreshing local artifacts after an update, or if something got messed up." echo "" + echo " pgo_build [instrument|profile|merge|optimize]" + echo " Build Firecracker with Profile-Guided Optimization (PGO). See README-pgo.md for more information" + echo "" cat <]] @@ -1083,6 +1086,49 @@ cmd_build_ci_artifacts() { cmd_fix_perms } +cmd_pgo_build() { + # For error checking and detecting architecture type + set -e + + PROFDATA_DIR=/tmp/firecracker-profdata + PROFDATA_FILE=/tmp/firecracker.profdata + + arch=$(uname -m) + if [[ "$arch" != "x86_64" ]]; then + echo "Warning: Non-x86_64 architecture detected ($arch)" + echo "Some features like /dev/kvm and full microVM profiling may not work." + fi + + + case "$1" in + instrument) + echo "Building instrumented Firecracker binary" + RUSTFLAGS="-Cprofile-generate=/tmp/firecracker-profdata" cargo build --release + echo "Instrumentation complete." + ;; + profile) + echo "Run workloads manually to generate .profraw files in /tmp/firecracker-profdata/" + echo "Please consult README-pgo.md for more information." + ;; + merge) + echo "Merging .profraw files" + if ! llvm-profdata merge -output=${PROFDATA_FILE} ${PROFDATA_DIR}/*.profraw; then + echo "Error: Failed to merge profile data." + echo " Make sure .profraw files exist and are readable." + exit 1 + fi + echo "Merging complete." + ;; + optimize) + echo "Building optimized Firecracker with profile data" + RUSTFLAGS="-Cprofile-use=/tmp/firecracker.profdata -Cllvm-args=-pgo-warn-missing-function" cargo build --release + ;; + *) + echo "Usage: $0 pgo_build [instrument|profile|merge|optimize]" + exit 1 + ;; + esac +} main() {