Merge pull request #3950 from swagatbora90/blkio-options

Add support for remaining blkio settings

Author: AkihiroSuda

cmd/nerdctl/container/container_create.go

@@ -199,19 +199,42 @@ func createOptions(cmd *cobra.Command) (types.ContainerCreateOptions, error) {
 	if err != nil {
 		return opt, err
 	}
+	opt.Cgroupns, err = cmd.Flags().GetString("cgroupns")
+	if err != nil {
+		return opt, err
+	}
+	opt.CgroupParent, err = cmd.Flags().GetString("cgroup-parent")
+	if err != nil {
+		return opt, err
+	}
+	opt.Device, err = cmd.Flags().GetStringSlice("device")
+	if err != nil {
+		return opt, err
+	}
+	// #endregion
+
+	// #region for blkio flags
 	opt.BlkioWeight, err = cmd.Flags().GetUint16("blkio-weight")
 	if err != nil {
 		return opt, err
 	}
-	opt.Cgroupns, err = cmd.Flags().GetString("cgroupns")
+	opt.BlkioWeightDevice, err = cmd.Flags().GetStringArray("blkio-weight-device")
 	if err != nil {
 		return opt, err
 	}
-	opt.CgroupParent, err = cmd.Flags().GetString("cgroup-parent")
+	opt.BlkioDeviceReadBps, err = cmd.Flags().GetStringArray("device-read-bps")
 	if err != nil {
 		return opt, err
 	}
-	opt.Device, err = cmd.Flags().GetStringSlice("device")
+	opt.BlkioDeviceWriteBps, err = cmd.Flags().GetStringArray("device-write-bps")
+	if err != nil {
+		return opt, err
+	}
+	opt.BlkioDeviceReadIOps, err = cmd.Flags().GetStringArray("device-read-iops")
+	if err != nil {
+		return opt, err
+	}
+	opt.BlkioDeviceWriteIOps, err = cmd.Flags().GetStringArray("device-write-iops")
 	if err != nil {
 		return opt, err
 	}

cmd/nerdctl/container/container_inspect_linux_test.go

@@ -19,6 +19,7 @@ package container
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"slices"
 	"strings"
 	"testing"
@@ -246,7 +247,6 @@ func TestContainerInspectHostConfig(t *testing.T) {
 	base.Cmd("run", "-d", "--name", testContainer,
 		"--cpuset-cpus", "0-1",
 		"--cpuset-mems", "0",
-		"--blkio-weight", "500",
 		"--cpu-shares", "1024",
 		"--cpu-quota", "100000",
 		"--group-add", "1000",
@@ -266,7 +266,6 @@ func TestContainerInspectHostConfig(t *testing.T) {
 
 	assert.Equal(t, "0-1", inspect.HostConfig.CPUSetCPUs)
 	assert.Equal(t, "0", inspect.HostConfig.CPUSetMems)
-	assert.Equal(t, uint16(500), inspect.HostConfig.BlkioWeight)
 	assert.Equal(t, uint64(1024), inspect.HostConfig.CPUShares)
 	assert.Equal(t, int64(100000), inspect.HostConfig.CPUQuota)
 	assert.Assert(t, slices.Contains(inspect.HostConfig.GroupAdd, "1000"), "Expected '1000' to be in GroupAdd")
@@ -311,6 +310,11 @@ func TestContainerInspectHostConfigDefaults(t *testing.T) {
 	assert.Equal(t, "", inspect.HostConfig.CPUSetCPUs)
 	assert.Equal(t, "", inspect.HostConfig.CPUSetMems)
 	assert.Equal(t, uint16(0), inspect.HostConfig.BlkioWeight)
+	assert.Equal(t, 0, len(inspect.HostConfig.BlkioWeightDevice))
+	assert.Equal(t, 0, len(inspect.HostConfig.BlkioDeviceReadBps))
+	assert.Equal(t, 0, len(inspect.HostConfig.BlkioDeviceReadIOps))
+	assert.Equal(t, 0, len(inspect.HostConfig.BlkioDeviceWriteBps))
+	assert.Equal(t, 0, len(inspect.HostConfig.BlkioDeviceWriteIOps))
 	assert.Equal(t, uint64(0), inspect.HostConfig.CPUShares)
 	assert.Equal(t, int64(0), inspect.HostConfig.CPUQuota)
 	assert.Equal(t, hc.GroupAddSize, len(inspect.HostConfig.GroupAdd))
@@ -456,6 +460,60 @@ func TestContainerInspectDevices(t *testing.T) {
 	assert.DeepEqual(t, expectedDevices, inspect.HostConfig.Devices)
 }
 
+func TestContainerInspectBlkioSettings(t *testing.T) {
+	testutil.DockerIncompatible(t)
+	testContainer := testutil.Identifier(t)
+	// Some of the blkio settings are not supported in cgroup v1.
+	// So skip this test if running on cgroup v1
+	if infoutil.CgroupsVersion() == "1" {
+		t.Skip("test skipped for rootless containers or if running with cgroup v1")
+	}
+
+	if rootlessutil.IsRootless() {
+		t.Skip("test requires root privilege to create a dummy device")
+	}
+
+	devPath := "/dev/dummy-zero"
+	// a dummy zero device: mknod /dev/dummy-zero c 1 5
+	helperCmd := exec.Command("mknod", []string{devPath, "c", "1", "5"}...)
+	if out, err := helperCmd.CombinedOutput(); err != nil {
+		err = fmt.Errorf("cannot create %q: %q: %w", devPath, string(out), err)
+		t.Fatal(err)
+	}
+
+	// ensure the file will be removed in case of failed in the test
+	defer func() {
+		if err := exec.Command("rm", "-f", devPath).Run(); err != nil {
+			t.Logf("failed to remove device %s: %v", devPath, err)
+		}
+	}()
+
+	base := testutil.NewBase(t)
+	defer base.Cmd("rm", "-f", testContainer).AssertOK()
+
+	base.Cmd("run", "-d", "--name", testContainer,
+		"--blkio-weight", "500",
+		"--blkio-weight-device", "/dev/dummy-zero:500",
+		"--device-read-bps", "/dev/dummy-zero:1048576",
+		"--device-read-iops", "/dev/dummy-zero:1000",
+		"--device-write-bps", "/dev/dummy-zero:2097152",
+		"--device-write-iops", "/dev/dummy-zero:2000",
+		testutil.AlpineImage, "sleep", "infinity").AssertOK()
+
+	inspect := base.InspectContainer(testContainer)
+	assert.Equal(t, uint16(500), inspect.HostConfig.BlkioWeight)
+	assert.Equal(t, 1, len(inspect.HostConfig.BlkioWeightDevice))
+	assert.Equal(t, uint16(500), *inspect.HostConfig.BlkioWeightDevice[0].Weight)
+	assert.Equal(t, 1, len(inspect.HostConfig.BlkioDeviceReadBps))
+	assert.Equal(t, uint64(1048576), inspect.HostConfig.BlkioDeviceReadBps[0].Rate)
+	assert.Equal(t, 1, len(inspect.HostConfig.BlkioDeviceWriteBps))
+	assert.Equal(t, uint64(2097152), inspect.HostConfig.BlkioDeviceWriteBps[0].Rate)
+	assert.Equal(t, 1, len(inspect.HostConfig.BlkioDeviceReadIOps))
+	assert.Equal(t, uint64(1000), inspect.HostConfig.BlkioDeviceReadIOps[0].Rate)
+	assert.Equal(t, 1, len(inspect.HostConfig.BlkioDeviceWriteIOps))
+	assert.Equal(t, uint64(2000), inspect.HostConfig.BlkioDeviceWriteIOps[0].Rate)
+}
+
 type hostConfigValues struct {
 	Driver       string
 	ShmSize      int64

cmd/nerdctl/container/container_run.go

@@ -155,7 +155,6 @@ func setCreateFlags(cmd *cobra.Command) {
 	})
 	cmd.Flags().Int64("pids-limit", -1, "Tune container pids limit (set -1 for unlimited)")
 	cmd.Flags().StringSlice("cgroup-conf", nil, "Configure cgroup v2 (key=value)")
-	cmd.Flags().Uint16("blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)")
 	cmd.Flags().String("cgroupns", defaults.CgroupnsMode(), `Cgroup namespace to use, the default depends on the cgroup version ("host"|"private")`)
 	cmd.Flags().String("cgroup-parent", "", "Optional parent cgroup for the container")
 	cmd.RegisterFlagCompletionFunc("cgroupns", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
@@ -173,6 +172,15 @@ func setCreateFlags(cmd *cobra.Command) {
 	cmd.Flags().String("rdt-class", "", "Name of the RDT class (or CLOS) to associate the container with")
 	// #endregion
 
+	// #region blkio flags
+	cmd.Flags().Uint16("blkio-weight", 0, "Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)")
+	cmd.Flags().StringArray("blkio-weight-device", []string{}, "Block IO weight (relative device weight) (default [])")
+	cmd.Flags().StringArray("device-read-bps", []string{}, "Limit read rate (bytes per second) from a device (default [])")
+	cmd.Flags().StringArray("device-read-iops", []string{}, "Limit read rate (IO per second) from a device (default [])")
+	cmd.Flags().StringArray("device-write-bps", []string{}, "Limit write rate (bytes per second) to a device (default [])")
+	cmd.Flags().StringArray("device-write-iops", []string{}, "Limit write rate (IO per second) to a device (default [])")
+	// #endregion
+
 	// user flags
 	cmd.Flags().StringP("user", "u", "", "Username or UID (format: <name|uid>[:<group|gid>])")
 	cmd.Flags().String("umask", "", "Set the umask inside the container. Defaults to 0022")

cmd/nerdctl/container/container_run_cgroup_linux_test.go

@@ -21,8 +21,10 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
@@ -477,3 +479,192 @@ func TestRunBlkioWeightCgroupV2(t *testing.T) {
 	base.Cmd("update", containerName, "--blkio-weight", "400").AssertOK()
 	base.Cmd("exec", containerName, "cat", "io.bfq.weight").AssertOutExactly("default 400\n")
 }
+
+func TestRunBlkioSettingCgroupV2(t *testing.T) {
+	testCase := nerdtest.Setup()
+	testCase.Require = nerdtest.Rootful
+
+	// Create dummy device path
+	dummyDev := "/dev/dummy-zero"
+
+	testCase.Setup = func(data test.Data, helpers test.Helpers) {
+		// Create dummy device
+		helperCmd := exec.Command("mknod", dummyDev, "c", "1", "5")
+		if out, err := helperCmd.CombinedOutput(); err != nil {
+			t.Fatalf("cannot create %q: %q: %v", dummyDev, string(out), err)
+		}
+	}
+
+	testCase.Cleanup = func(data test.Data, helpers test.Helpers) {
+		// Clean up the dummy device
+		if err := exec.Command("rm", "-f", dummyDev).Run(); err != nil {
+			t.Logf("failed to remove device %s: %v", dummyDev, err)
+		}
+	}
+
+	testCase.SubTests = []*test.Case{
+		{
+			Description: "blkio-weight",
+			Require:     nerdtest.CGroupV2,
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--blkio-weight", "150",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							assert.Assert(t, strings.Contains(helpers.Capture("inspect", "--format", "{{.HostConfig.BlkioWeight}}", data.Identifier()), "150"))
+						},
+					),
+				}
+			},
+		},
+		{
+			Description: "blkio-weight-device",
+			Require:     nerdtest.CGroupV2,
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--blkio-weight-device", dummyDev+":100",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							inspectOut := helpers.Capture("inspect", "--format", "{{range .HostConfig.BlkioWeightDevice}}{{.Weight}}{{end}}", data.Identifier())
+							assert.Assert(t, strings.Contains(inspectOut, "100"))
+						},
+					),
+				}
+			},
+		},
+		{
+			Description: "device-read-bps",
+			Require: require.All(
+				nerdtest.CGroupV2,
+				// Docker cli (v26.1.3) available in github runners has a bug where some of the blkio options
+				// do not work https://github.com/docker/cli/issues/5321. The fix has been merged to the latest releases
+				// but not currently available in the v26 release.
+				require.Not(nerdtest.Docker),
+			),
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--device-read-bps", dummyDev+":1048576",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							inspectOut := helpers.Capture("inspect", "--format", "{{range .HostConfig.BlkioDeviceReadBps}}{{.Rate}}{{end}}", data.Identifier())
+							assert.Assert(t, strings.Contains(inspectOut, "1048576"))
+						},
+					),
+				}
+			},
+		},
+		{
+			Description: "device-write-bps",
+			Require: require.All(
+				nerdtest.CGroupV2,
+				// Docker cli (v26.1.3) available in github runners has a bug where some of the blkio options
+				// do not work https://github.com/docker/cli/issues/5321. The fix has been merged to the latest releases
+				// but not currently available in the v26 release.
+				require.Not(nerdtest.Docker),
+			),
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--device-write-bps", dummyDev+":2097152",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							inspectOut := helpers.Capture("inspect", "--format", "{{range .HostConfig.BlkioDeviceWriteBps}}{{.Rate}}{{end}}", data.Identifier())
+							assert.Assert(t, strings.Contains(inspectOut, "2097152"))
+						},
+					),
+				}
+			},
+		},
+		{
+			Description: "device-read-iops",
+			Require: require.All(
+				nerdtest.CGroupV2,
+				// Docker cli (v26.1.3) available in github runners has a bug where some of the blkio options
+				// do not work https://github.com/docker/cli/issues/5321. The fix has been merged to the latest releases
+				// but not currently available in the v26 release.
+				require.Not(nerdtest.Docker),
+			),
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--device-read-iops", dummyDev+":1000",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							inspectOut := helpers.Capture("inspect", "--format", "{{range .HostConfig.BlkioDeviceReadIOps}}{{.Rate}}{{end}}", data.Identifier())
+							assert.Assert(t, strings.Contains(inspectOut, "1000"))
+						},
+					),
+				}
+			},
+		},
+		{
+			Description: "device-write-iops",
+			Require: require.All(
+				nerdtest.CGroupV2,
+				// Docker cli (v26.1.3) available in github runners has a bug where some of the blkio options
+				// do not work https://github.com/docker/cli/issues/5321. The fix has been merged to the latest releases
+				// but not currently available in the v26 release.
+				require.Not(nerdtest.Docker),
+			),
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("run", "-d", "--name", data.Identifier(),
+					"--device-write-iops", dummyDev+":2000",
+					testutil.AlpineImage, "sleep", "infinity")
+			},
+			Cleanup: func(data test.Data, helpers test.Helpers) {
+				helpers.Anyhow("rm", "-f", data.Identifier())
+			},
+			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
+				return &test.Expected{
+					ExitCode: 0,
+					Output: expect.All(
+						func(stdout string, info string, t *testing.T) {
+							inspectOut := helpers.Capture("inspect", "--format", "{{range .HostConfig.BlkioDeviceWriteIOps}}{{.Rate}}{{end}}", data.Identifier())
+							assert.Assert(t, strings.Contains(inspectOut, "2000"))
+						},
+					),
+				}
+			},
+		},
+	}
+
+	testCase.Run(t)
+}

docs/command-reference.md

@@ -213,6 +213,11 @@ Resource flags:
 - :whale: `--pids-limit`: Tune container pids limit
 - :nerd_face: `--cgroup-conf`: Configure cgroup v2 (key=value)
 - :whale: `--blkio-weight`: Block IO (relative weight), between 10 and 1000, or 0 to disable (default 0)
+- :whale: `--blkio-weight-device`: Block IO weight (relative device weight)
+- :whale: `--device-read-bps`: Limit read rate (bytes per second) from a device
+- :whale: `--device-read-iops`: Limit read rate (IO per second) from a device
+- :whale: `--device-write-bps`: Limit write rate (bytes per second) to a device
+- :whale: `--device-write-iops`: Limit write rate (IO per second) to a device
 - :whale: `--cgroupns=(host|private)`: Cgroup namespace to use
   - Default: "private" on cgroup v2 hosts, "host" on cgroup v1 hosts
 - :whale: `--cgroup-parent`: Optional parent cgroup for the container
@@ -414,7 +419,7 @@ IPFS flags:
 - :nerd_face: `--ipfs-address`: Multiaddr of IPFS API (default uses `$IPFS_PATH` env variable if defined or local directory `~/.ipfs`)
 
 Unimplemented `docker run` flags:
-    `--blkio-weight-device`, `--cpu-rt-*`, `--device-*`,
+    `--cpu-rt-*`, `--device-cgroup-rule`,
     `--disable-content-trust`, `--expose`, `--health-*`, `--isolation`, `--no-healthcheck`,
     `--link*`, `--publish-all`, `--storage-opt`,
     `--userns`, `--volume-driver`