From 9c40cf8573b18a1aa40d90a0345de5a8f7489a0a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 1 Mar 2024 15:06:48 +0100 Subject: [PATCH] add e2e tests for spoc --- .github/workflows/test.yml | 15 ++ .gitignore | 3 + Makefile | 4 + .../.snapshots/TestSpoc-record-AppArmor-files | 31 ++++ .../TestSpoc-record-AppArmor-sockets | 48 ++++++ .../TestSpoc-record-AppArmor-subprocess | 35 +++++ test/spoc/demobinary.go | 142 ++++++++++++++++++ test/spoc/e2e_spoc_test.go | 101 +++++++++++++ 8 files changed, 379 insertions(+) create mode 100644 test/spoc/.snapshots/TestSpoc-record-AppArmor-files create mode 100644 test/spoc/.snapshots/TestSpoc-record-AppArmor-sockets create mode 100644 test/spoc/.snapshots/TestSpoc-record-AppArmor-subprocess create mode 100644 test/spoc/demobinary.go create mode 100644 test/spoc/e2e_spoc_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44261dcbab..897c12fada 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,6 +166,21 @@ jobs: $RUN "echo \"export E2E_TEST_FLAKY_TESTS_ONLY=true\" >> /vagrant/hack/ci/env-flatcar.sh" $RUN hack/ci/e2e-flatcar-dev-container.sh + e2e-spoc: + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v3.5.2 + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y libseccomp-dev libelf-dev libapparmor-dev + sudo hack/install-libbpf.sh + - run: make test-spoc-e2e + baseprofiles: needs: image runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index b03bd2565b..81b7dfc297 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ Vagrantfile build.tar.gz image.tar result +/test/spoc/demobinary +/test/spoc/demobinary-child + diff --git a/Makefile b/Makefile index a5b3fd9be5..ef91da4a8d 100644 --- a/Makefile +++ b/Makefile @@ -469,6 +469,10 @@ test-flaky-e2e: ## Only run the flaky end-to-end tests E2E_SKIP_FLAKY_TESTS=false \ $(GO) test -parallel 1 -timeout 20m -count=1 ./test/... -v -testify.m '^(TestSecurityProfilesOperator_Flaky)$$' +.PHONY: test-spoc-e2e +test-spoc-e2e: build/spoc + $(GO) test -v ./test/spoc/... + # Generate CRD manifests manifests: $(BUILD_DIR)/kubernetes-split-yaml $(BUILD_DIR)/kustomize ./hack/sort-crds.sh "$(CONTROLLER_GEN_CMD) $(CRD_OPTIONS) paths='./api/spod/...' output:crd:stdout" "deploy/base-crds/crds/securityprofilesoperatordaemon.yaml" diff --git a/test/spoc/.snapshots/TestSpoc-record-AppArmor-files b/test/spoc/.snapshots/TestSpoc-record-AppArmor-files new file mode 100644 index 0000000000..e2057e9470 --- /dev/null +++ b/test/spoc/.snapshots/TestSpoc-record-AppArmor-files @@ -0,0 +1,31 @@ +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + capability: + allowedCapabilities: + - dac_read_search + executable: {} + filesystem: + readOnlyPaths: + - ../../README.md + writeOnlyPaths: + - /dev/null +status: {} + +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + executable: {} + filesystem: + readWritePaths: + - /dev/null +status: {} + diff --git a/test/spoc/.snapshots/TestSpoc-record-AppArmor-sockets b/test/spoc/.snapshots/TestSpoc-record-AppArmor-sockets new file mode 100644 index 0000000000..7ffed88bb9 --- /dev/null +++ b/test/spoc/.snapshots/TestSpoc-record-AppArmor-sockets @@ -0,0 +1,48 @@ +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + capability: + allowedCapabilities: + - net_admin + executable: {} + filesystem: + readOnlyPaths: + - /proc/@{pid}/sys/net/core/somaxconn + network: + allowedProtocols: + allowTcp: true +status: {} + +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + executable: {} + network: + allowedProtocols: + allowTcp: true + allowUdp: true +status: {} + +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + capability: + allowedCapabilities: + - net_raw + executable: {} + network: + allowRaw: true +status: {} + diff --git a/test/spoc/.snapshots/TestSpoc-record-AppArmor-subprocess b/test/spoc/.snapshots/TestSpoc-record-AppArmor-subprocess new file mode 100644 index 0000000000..4066e2a289 --- /dev/null +++ b/test/spoc/.snapshots/TestSpoc-record-AppArmor-subprocess @@ -0,0 +1,35 @@ +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + capability: + allowedCapabilities: + - dac_read_search + - dac_override + executable: + allowedExecutables: + - ./demobinary-child +status: {} + +apiVersion: security-profiles-operator.x-k8s.io/v1alpha1 +kind: AppArmorProfile +metadata: + creationTimestamp: null + name: demobinary +spec: + abstract: + capability: + allowedCapabilities: + - dac_read_search + - dac_override + executable: + allowedExecutables: + - ./demobinary + filesystem: + readOnlyPaths: + - /dev/null +status: {} + diff --git a/test/spoc/demobinary.go b/test/spoc/demobinary.go new file mode 100644 index 0000000000..f99e077ace --- /dev/null +++ b/test/spoc/demobinary.go @@ -0,0 +1,142 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// #cgo LDFLAGS: -ldl +// #include +import "C" + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +const LogPrefixEnvVar = "LOGPREFIX" + +// Demo binary to exercise various capabilities that may be restricted by seccomp/apparmor. +func main() { + log.SetPrefix(fmt.Sprintf("%s[pid:%d] ", os.Getenv(LogPrefixEnvVar), os.Getpid())) + log.SetFlags(log.Lshortfile) + log.Println("⏩", os.Args) + + fileWrite := flag.String("file-write", "", "write file (e.g. /dev/null)") + fileRead := flag.String("file-read", "", "read file (e.g. /dev/null)") + fileSymlink := flag.String("file-symlink", "", "Create symlink using the following syntax: OLD:NEW") + netTCP := flag.Bool("net-tcp", false, "spawn a tcp server") + netUDP := flag.Bool("net-udp", false, "spawn a udp server") + netIcmp := flag.Bool("net-icmp", false, "open an icmp socket, exercise NET_RAW capability.") + library := flag.String("load-library", "", "load a shared library") + sleep := flag.Int("sleep", 0, "sleep N seconds before exiting.") + crash := flag.Bool("crash", false, "crash instead of exiting.") + + flag.Parse() + + subprocess := flag.Args() + + if *fileWrite != "" { + const fileMode = 0o666 + err := os.WriteFile(*fileWrite, []byte{}, fileMode) + if err != nil { + log.Fatal("❌ Error creating file:", err) + } + log.Println("✅ File write successful:", *fileWrite) + // make file writable for other users so that sudo/non-sudo testing works. + err = os.Chmod(*fileWrite, fileMode) + if err != nil { + log.Fatal("❌ Error setting file permissions:", err) + } + } + if *fileSymlink != "" { + oldname, newname, found := strings.Cut(*fileSymlink, ":") + if !found { + log.Fatal("❌ Symlink syntax: OLD:NEW") + } + err := os.Symlink(oldname, newname) + if err != nil { + log.Fatal("❌ Error creating symlink:", err) + } + log.Println("✅ Symlink created:", newname, "->", oldname) + } + if *fileRead != "" { + _, err := os.ReadFile(*fileRead) + if err != nil { + log.Fatal("❌ Error reading file:", err) + } + log.Println("✅ File read successful:", *fileRead) + } + if *netTCP { + listener, err := net.Listen("tcp", ":0") + if err != nil { + log.Fatal("❌ Error starting TCP server:", err) + } + log.Println("✅ TCP server spawned:", listener.Addr()) + defer listener.Close() + } + if *netUDP { + server, err := net.ListenPacket("udp", ":0") + if err != nil { + //nolint:gocritic // gocritic is terminally confused here. + log.Fatal("❌ Error starting UDP server:", err) + } + log.Println("✅ UDP server spawned:", server.LocalAddr()) + defer server.Close() + } + if *netIcmp { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_ICMP) + if err != nil { + log.Fatal("❌ Error opening ICMP socket:", err) + } + log.Println("✅ ICMP socket opened: fd", fd) + defer syscall.Close(fd) + } + if len(subprocess) > 0 { + cmd := exec.Command(subprocess[0], subprocess[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), "LOGPREFIX=\t"+os.Getenv(LogPrefixEnvVar)) + err := cmd.Run() + if err != nil { + log.Fatal("❌ Error running subprocess:", err) + } + log.Println("✅ Subprocess ran successfully:", subprocess) + } + if *library != "" { + if handle := C.dlopen(C.CString(*library), C.RTLD_NOW); handle == nil { + log.Fatal("❌ Error loading library: ", C.GoString(C.dlerror())) + } + log.Println("✅ Library loaded successfully:", *library) + } + if *sleep > 0 { + log.Println("⏳ Sleeping for", *sleep, "seconds...") + time.Sleep(time.Duration(*sleep) * time.Second) + } + if *crash { + log.Println("🫡 Terminating with SIGKILL...") + err := syscall.Kill(syscall.Getpid(), syscall.SIGKILL) + if err != nil { + log.Fatal("❌ Error sending SIGKILL:", err) + } + } + log.Println("⭐️ Success.") +} diff --git a/test/spoc/e2e_spoc_test.go b/test/spoc/e2e_spoc_test.go new file mode 100644 index 0000000000..1bdfd4c32b --- /dev/null +++ b/test/spoc/e2e_spoc_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main_test + +import ( + "os" + "os/exec" + "testing" + + "github.com/bradleyjkemp/cupaloy" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "sigs.k8s.io/release-utils/util" +) + +//nolint:paralleltest // should not run in parallel +func TestSpoc(t *testing.T) { + logrus.Info("Building demobinary...") + cmd := exec.Command("go", "build", "demobinary.go") + err := cmd.Run() + require.Nil(t, err, "failed to build demobinary.go") + err = util.CopyFileLocal("demobinary", "demobinary-child", true) + require.Nil(t, err) + err = os.Chmod("demobinary-child", 0o700) + require.Nil(t, err) + + t.Run("record", recordTest) +} + +func recordTest(t *testing.T) { + t.Run("AppArmor", recordAppArmorTest) + t.Run("Seccomp", recordSeccompTest) +} + +func recordAppArmorTest(t *testing.T) { + t.Run("files", func(t *testing.T) { + cupaloy.SnapshotT( + t, + // this still shows a bug: the path to README should be resolved. + record(t, "apparmor", "--file-read", "../../README.md", "--file-write", "/dev/null"), + record(t, "apparmor", "--file-read", "/dev/null", "--file-write", "/dev/null"), + ) + }) + t.Run("sockets", func(t *testing.T) { + cupaloy.SnapshotT( + t, + record(t, "apparmor", "--net-tcp"), + // Go is doing Go things and does some TCP syscalls when opening a UDP socket. + // So the snapshot looks a bit weird here. + record(t, "apparmor", "--net-udp"), + record(t, "apparmor", "--net-icmp"), + ) + }) + + t.Run("subprocess", func(t *testing.T) { + // Ensure that we can run subprocesses and that their action are recorded, too. + cupaloy.SnapshotT( + t, + record(t, "apparmor", "./demobinary-child", "--file-read", "/dev/null"), + record(t, "apparmor", "./demobinary", "--file-read", "/dev/null"), + ) + }) +} + +func recordSeccompTest(t *testing.T) { + // smoke test for seccomp + record(t, "seccomp", "./demobinary-child", "--net-tcp") + // TODO: add snapshot testing here - currently disabled because flaky +} + +func runSpoc(t *testing.T, args ...string) []byte { + t.Helper() + args = append([]string{"../../build/spoc"}, args...) + cmd := exec.Command("sudo", args...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + require.Nil(t, err, "failed to run spoc") + return out +} + +func record(t *testing.T, typ string, args ...string) []byte { + t.Helper() + args = append([]string{ + "record", "-t", typ, "-o", "/dev/stdout", "--no-base-syscalls", "./demobinary", + }, args...) + return runSpoc(t, args...) +}