Skip to content

Commit 1cb9072

Browse files
JamesC1305JamesC1305
committed
Create performance tests for hotplug latency
Create performance tests that measure the latency of hotplugging using both a udev rule and an a userspace agent to online vCPUs. Signed-off-by: James Curtis <[email protected]>
1 parent 2029a08 commit 1cb9072

File tree

8 files changed

+259
-1
lines changed

8 files changed

+259
-1
lines changed

‎src/vmm/src/lib.rs

-1
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,6 @@ impl Vmm {
683683

684684
self.acpi_device_manager.notify_cpu_container()?;
685685

686-
#[cfg(test)]
687686
if let Some(devices::BusDevice::BootTimer(timer)) =
688687
self.get_bus_device(DeviceType::BootTimer, "BootTimer")
689688
{

‎tests/conftest.py

+3
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,9 @@ def rootfs_fxt(request, record_property):
377377
guest_kernel_linux_5_10 = pytest.fixture(
378378
guest_kernel_fxt, params=kernel_params("vmlinux-5.10*")
379379
)
380+
guest_kernel_linux_acpi_only = pytest.fixture(
381+
guest_kernel_fxt, params=kernel_params("vmlinux-5.10.219")
382+
)
380383
# Use the unfiltered selector, since we don't officially support 6.1 yet.
381384
# TODO: switch to default selector once we add full 6.1 support.
382385
guest_kernel_linux_6_1 = pytest.fixture(

‎tests/framework/http_api.py

+1
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,4 @@ def __init__(self, api_usocket_full_name):
123123
self.snapshot_load = Resource(self, "/snapshot/load")
124124
self.cpu_config = Resource(self, "/cpu-config")
125125
self.entropy = Resource(self, "/entropy")
126+
self.hotplug = Resource(self, "/hotplug")

‎tests/host_tools/1-cpu-hotplug.rules

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SUBSYSTEM=="cpu", ACTION=="add", ATTR{online}!="1", ATTR{online}="1"

‎tests/host_tools/hotplug.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
while :; do
6+
[[ -d /sys/devices/system/cpu/cpu$1 ]] && break
7+
done
8+
9+
for i in $(seq 1 $1); do
10+
echo 1 >/sys/devices/system/cpu/cpu$i/online
11+
done
12+
13+
while :; do
14+
[[ $(nproc) == $((1 + $1)) ]] && break
15+
done
16+
17+
/home/hotplug_time.o

‎tests/host_tools/hotplug_time.c

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Init wrapper for boot timing. It points at /sbin/init.
5+
6+
#include <fcntl.h>
7+
#include <sys/mman.h>
8+
#include <sys/types.h>
9+
#include <unistd.h>
10+
11+
// Base address values are defined in arch/src/lib.rs as arch::MMIO_MEM_START.
12+
// Values are computed in arch/src/<arch>/mod.rs from the architecture layouts.
13+
// Position on the bus is defined by MMIO_LEN increments, where MMIO_LEN is
14+
// defined as 0x1000 in vmm/src/device_manager/mmio.rs.
15+
#ifdef __x86_64__
16+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0xd0000000
17+
#endif
18+
#ifdef __aarch64__
19+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0x40000000
20+
#endif
21+
22+
#define MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE 123
23+
24+
int main() {
25+
int fd = open("/dev/mem", (O_RDWR | O_SYNC | O_CLOEXEC));
26+
int mapped_size = getpagesize();
27+
28+
char *map_base = mmap(NULL, mapped_size, PROT_WRITE, MAP_SHARED, fd,
29+
MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE);
30+
31+
*map_base = MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE;
32+
msync(map_base, mapped_size, MS_ASYNC);
33+
}

‎tests/host_tools/hotplug_udev.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
while :; do
6+
[[ $(nproc) == $((1 + $1)) ]] && break
7+
done
8+
9+
/home/hotplug_time.o

‎tests/integration_tests/performance/test_vcpu_hotplug.py

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Testing hotplug performance"""
5+
6+
import os
7+
import platform
8+
import re
9+
import time
10+
from pathlib import Path
11+
12+
import pandas
13+
import pytest
14+
15+
from framework.utils_cpuid import check_guest_cpuid_output
16+
from host_tools.cargo_build import gcc_compile
17+
18+
19+
@pytest.mark.nonci
20+
@pytest.mark.skipif(
21+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
22+
)
23+
@pytest.mark.parametrize(
24+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
25+
)
26+
def test_custom_udev_rule_latency(
27+
microvm_factory,
28+
guest_kernel_linux_6_1,
29+
rootfs_rw,
30+
vcpu_count,
31+
results_dir,
32+
test_fc_session_root_path,
33+
):
34+
"""Test the latency for hotplugging and booting CPUs in the guest"""
35+
hotplug_time_path = os.path.join(test_fc_session_root_path, "hotplug_time.o")
36+
gcc_compile(Path("./host_tools/hotplug_time.c"), hotplug_time_path)
37+
data = []
38+
for _ in range(20):
39+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
40+
uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
41+
uvm_hotplug.help.enable_console()
42+
uvm_hotplug.spawn()
43+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
44+
uvm_hotplug.add_net_iface()
45+
uvm_hotplug.start()
46+
uvm_hotplug.ssh.scp_put(
47+
Path("./host_tools/hotplug_udev.sh"), Path("/home/hotplug_udev.sh")
48+
)
49+
uvm_hotplug.ssh.scp_put(hotplug_time_path, Path("/home/hotplug_time.o"))
50+
uvm_hotplug.ssh.scp_put(
51+
Path("./host_tools/1-cpu-hotplug.rules"),
52+
Path("/usr/lib/udev/rules.d/1-cpu-hotplug.rules"),
53+
)
54+
uvm_hotplug.ssh.run(
55+
f"udevadm control --reload-rules && tmux new-session -d /bin/bash /home/hotplug_udev.sh {vcpu_count}"
56+
)
57+
58+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
59+
time.sleep(5)
60+
61+
# Extract API call duration
62+
api_duration = (
63+
float(
64+
re.findall(
65+
r"Total previous API call duration: (\d+) us\.",
66+
uvm_hotplug.log_data,
67+
)[-1]
68+
)
69+
/ 1000
70+
)
71+
try:
72+
timestamp = (
73+
float(
74+
re.findall(
75+
r"Guest-boot-time\s+\=\s+(\d+)\s+us", uvm_hotplug.log_data
76+
)[0]
77+
)
78+
/ 1000
79+
)
80+
except IndexError:
81+
uvm_hotplug.kill()
82+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
83+
continue
84+
85+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
86+
87+
check_guest_cpuid_output(
88+
uvm_hotplug,
89+
"lscpu",
90+
None,
91+
":",
92+
{
93+
"CPU(s)": str(1 + vcpu_count),
94+
"On-line CPU(s) list": f"0-{vcpu_count}",
95+
},
96+
)
97+
uvm_hotplug.kill()
98+
99+
output_file = results_dir / f"hotplug-{vcpu_count}.csv"
100+
101+
csv_data = pandas.DataFrame.from_dict(data).to_csv(
102+
index=False,
103+
float_format="%.3f",
104+
)
105+
106+
output_file.write_text(csv_data)
107+
108+
109+
@pytest.mark.nonci
110+
@pytest.mark.skipif(
111+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
112+
)
113+
@pytest.mark.parametrize(
114+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
115+
)
116+
def test_manual_latency(
117+
microvm_factory,
118+
guest_kernel_linux_6_1,
119+
rootfs_rw,
120+
vcpu_count,
121+
results_dir,
122+
test_fc_session_root_path,
123+
):
124+
"""Test the latency for hotplugging and booting CPUs in the guest"""
125+
126+
hotplug_time_path = os.path.join(test_fc_session_root_path, "hotplug_time.o")
127+
gcc_compile(Path("./host_tools/hotplug_time.c"), hotplug_time_path)
128+
data = []
129+
for _ in range(20):
130+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_6_1, rootfs_rw)
131+
uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
132+
uvm_hotplug.help.enable_console()
133+
uvm_hotplug.spawn()
134+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
135+
uvm_hotplug.add_net_iface()
136+
uvm_hotplug.start()
137+
138+
uvm_hotplug.ssh.scp_put(
139+
Path("./host_tools/hotplug.sh"), Path("/home/hotplug.sh")
140+
)
141+
uvm_hotplug.ssh.scp_put(hotplug_time_path, Path("/home/hotplug_time.o"))
142+
uvm_hotplug.ssh.run(
143+
f"tmux new-session -d /bin/bash /home/hotplug.sh {vcpu_count}"
144+
)
145+
146+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
147+
148+
time.sleep(5)
149+
# Extract API call duration
150+
api_duration = (
151+
float(
152+
re.findall(
153+
r"Total previous API call duration: (\d+) us\.",
154+
uvm_hotplug.log_data,
155+
)[-1]
156+
)
157+
/ 1000
158+
)
159+
try:
160+
timestamp = (
161+
float(
162+
re.findall(
163+
r"Guest-boot-time\s+\=\s+(\d+)\s+us", uvm_hotplug.log_data
164+
)[0]
165+
)
166+
/ 1000
167+
)
168+
except IndexError:
169+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
170+
uvm_hotplug.kill()
171+
continue
172+
173+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
174+
175+
check_guest_cpuid_output(
176+
uvm_hotplug,
177+
"lscpu",
178+
None,
179+
":",
180+
{
181+
"CPU(s)": str(1 + vcpu_count),
182+
"On-line CPU(s) list": f"0-{vcpu_count}",
183+
},
184+
)
185+
186+
uvm_hotplug.kill()
187+
188+
output_file = results_dir / f"hotplug-{vcpu_count}.csv"
189+
190+
csv_data = pandas.DataFrame.from_dict(data).to_csv(
191+
index=False,
192+
float_format="%.3f",
193+
)
194+
195+
output_file.write_text(csv_data)

0 commit comments

Comments
 (0)