From bf21411e90327b4c3192f1c1debe9cb0b1c4dcb5 Mon Sep 17 00:00:00 2001 From: Dan Mihai Date: Fri, 25 Aug 2023 21:20:36 +0000 Subject: [PATCH] tests: add policy to k8s tests Use AGENT_POLICY=yes when building the Guest images, and add a permissive test policy to the k8s tests for: - CBL-Mariner - SEV - SNP - TDX Also, add an example of policy rejecting ExecProcessRequest. Fixes: #7667 Signed-off-by: Dan Mihai --- .../allow-all-except-exec-process.rego | 39 +++++++++ src/kata-opa/allow-all.rego | 2 +- .../kubernetes/k8s-exec-rejected.bats | 41 +++++++++ tests/integration/kubernetes/k8s-job.bats | 5 ++ .../integration/kubernetes/k8s-pod-quota.bats | 4 + .../kubernetes/run_kubernetes_tests.sh | 85 +++++++++++++++++-- tools/packaging/guest-image/build_image.sh | 6 +- .../local-build/kata-deploy-binaries.sh | 3 + 8 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 src/kata-opa/allow-all-except-exec-process.rego create mode 100644 tests/integration/kubernetes/k8s-exec-rejected.bats diff --git a/src/kata-opa/allow-all-except-exec-process.rego b/src/kata-opa/allow-all-except-exec-process.rego new file mode 100644 index 000000000000..7d77015b2408 --- /dev/null +++ b/src/kata-opa/allow-all-except-exec-process.rego @@ -0,0 +1,39 @@ +package agent_policy + +default AddARPNeighborsRequest := true +default AddSwapRequest := true +default CloseStdinRequest := true +default CopyFileRequest := true +default CreateContainerRequest := true +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default GetMetricsRequest := true +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default ListInterfacesRequest := true +default ListRoutesRequest := true +default MemHotplugByProbeRequest := true +default OnlineCPUMemRequest := true +default PauseContainerRequest := true +default PullImageRequest := true +default ReadStreamRequest := true +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default ReseedRandomDevRequest := false +default ResumeContainerRequest := true +default SetGuestDateTimeRequest := true +default SetPolicyRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StartTracingRequest := true +default StatsContainerRequest := true +default StopTracingRequest := true +default TtyWinResizeRequest := true +default UpdateContainerRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := true + +default ExecProcessRequest := false diff --git a/src/kata-opa/allow-all.rego b/src/kata-opa/allow-all.rego index 0a974ff6e6c2..cf813a600b26 100644 --- a/src/kata-opa/allow-all.rego +++ b/src/kata-opa/allow-all.rego @@ -7,7 +7,7 @@ default CopyFileRequest := true default CreateContainerRequest := true default CreateSandboxRequest := true default DestroySandboxRequest := true -default ExecProcessRequest = true +default ExecProcessRequest := true default GetMetricsRequest := true default GetOOMEventRequest := true default GuestDetailsRequest := true diff --git a/tests/integration/kubernetes/k8s-exec-rejected.bats b/tests/integration/kubernetes/k8s-exec-rejected.bats new file mode 100644 index 000000000000..a3b9d2bf9f0e --- /dev/null +++ b/tests/integration/kubernetes/k8s-exec-rejected.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2023 Microsoft. +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/../../common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +setup() { + get_pod_config_dir + pod_name="busybox" + pod_yaml="${pod_config_dir}/busybox-pod.yaml" + + # String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all-except-exec-process.rego" + allow_all_except_exec_policy="cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IFJlc3VtZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldEd1ZXN0RGF0ZVRpbWVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRQb2xpY3lSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTaWduYWxQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGF0c0NvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0b3BUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVHR5V2luUmVzaXplUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlRXBoZW1lcmFsTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlSW50ZXJmYWNlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlUm91dGVzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV2FpdFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXcml0ZVN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQoKZGVmYXVsdCBFeGVjUHJvY2Vzc1JlcXVlc3QgOj0gZmFsc2UK" +} + +@test "Kubectl exec rejected by policy" { + # Add to the YAML file a policy that rejects ExecProcessRequest. + yq write -i "${pod_yaml}" \ + 'metadata.annotations."io.katacontainers.config.agent.policy"' \ + "${allow_all_except_exec_policy}" + + # Create the pod + kubectl create -f "${pod_yaml}" + + # Wait for pod to start + kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name" + + # Try executing a command in the Pod - an action rejected by the agent policy. + kubectl exec "$pod_name" -- date 2>&1 | grep "ExecProcessRequest is blocked by policy" +} + +teardown() { + # Debugging information + kubectl describe "pod/$pod_name" + + kubectl delete pod "$pod_name" +} diff --git a/tests/integration/kubernetes/k8s-job.bats b/tests/integration/kubernetes/k8s-job.bats index e1fd3cc38571..1fbc672e2bb2 100644 --- a/tests/integration/kubernetes/k8s-job.bats +++ b/tests/integration/kubernetes/k8s-job.bats @@ -34,6 +34,11 @@ setup() { } teardown() { + # Debugging information + kubectl describe pod "$pod_name" + kubectl describe jobs/"$job_name" + + # Clean-up kubectl delete pod "$pod_name" # Verify that pod is not running run kubectl get pods diff --git a/tests/integration/kubernetes/k8s-pod-quota.bats b/tests/integration/kubernetes/k8s-pod-quota.bats index 20ad06274338..88840ec3684d 100644 --- a/tests/integration/kubernetes/k8s-pod-quota.bats +++ b/tests/integration/kubernetes/k8s-pod-quota.bats @@ -31,6 +31,10 @@ setup() { } teardown() { + # Debugging information + kubectl describe deployment ${deployment_name} + + # Clean-up kubectl delete -f "${pod_config_dir}/pod-quota-deployment.yaml" kubectl delete -f "${pod_config_dir}/resource-quota.yaml" } diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index 3d963c6c3d77..aef9f3a3d6aa 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -14,6 +14,9 @@ TARGET_ARCH="${TARGET_ARCH:-x86_64}" KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}" +# String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all.rego" +ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA9IHRydWUKZGVmYXVsdCBHZXRNZXRyaWNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBMaXN0SW50ZXJmYWNlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBNZW1Ib3RwbHVnQnlQcm9iZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFBhdXNlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUHVsbEltYWdlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVhZFN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlc2VlZFJhbmRvbURldlJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBSZXN1bWVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRHdWVzdERhdGVUaW1lUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0UG9saWN5UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdG9wVHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFR0eVdpblJlc2l6ZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUVwaGVtZXJhbE1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUludGVyZmFjZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZVJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdhaXRQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV3JpdGVTdHJlYW1SZXF1ZXN0IDo9IHRydWUK"}" + if [ -n "${K8S_TEST_UNION:-}" ]; then K8S_TEST_UNION=($K8S_TEST_UNION) else @@ -58,6 +61,74 @@ else ) fi +policy_tests_enabled() { + # The Guest images for these platforms have been built using AGENT_POLICY=yes - + # see kata-deploy-binaries.sh. + [ "${KATA_HYPERVISOR}" == "qemu-sev" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || \ + [ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HOST_OS}" == "cbl-mariner" ] +} + +add_policy_to_yaml() { + local yaml_file="$1" + local resource_kind="$(yq read ${yaml_file} kind)" + + case "${resource_kind}" in + + Pod) + echo "Adding policy to ${resource_kind} from ${yaml_file}" + ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${K8S_TEST_YAML}" \ + 'metadata.annotations."io.katacontainers.config.agent.policy"' \ + "${ALLOW_ALL_POLICY}" + ;; + + Deployment|Job|ReplicationController) + if [ "${KATA_HOST_OS}" == "cbl-mariner" ]; then + echo "Issue #7764: using policy for ${resource_kind} from ${yaml_file} is blocked on ${KATA_HOST_OS}" + else + echo "Adding policy to ${resource_kind} from ${yaml_file}" + ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${K8S_TEST_YAML}" \ + 'spec.template.metadata.annotations."io.katacontainers.config.agent.policy"' \ + "${ALLOW_ALL_POLICY}" + fi + ;; + + List) + echo "Issue #7765: adding policy to ${resource_kind} from ${yaml_file} is not implemented yet" + ;; + + ConfigMap|LimitRange|Namespace|PersistentVolume|PersistentVolumeClaim|RuntimeClass|Secret|Service) + echo "Policy is not required for ${resource_kind} from ${yaml_file}" + ;; + + *) + echo "k8s resource type ${resource_kind} from ${yaml_file} is not yet supported for policy testing" + return 1 + ;; + + esac +} + +add_policy_to_successful_tests() { + info "Add policy to test YAML files" + for K8S_TEST_YAML in runtimeclass_workloads_work/*.yaml + do + add_policy_to_yaml "${K8S_TEST_YAML}" + done +} + +test_successful_actions() { + info "Test actions that must be successful" + for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]} + do + bats "${K8S_TEST_ENTRY}" + done +} + +test_actions_rejected_by_policy() { + info "Test k8s actions that get rejected by policy" + bats k8s-exec-rejected.bats +} + # we may need to skip a few test cases when running on non-x86_64 arch arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml" if [ -f "${arch_config_file}" ]; then @@ -65,8 +136,12 @@ if [ -f "${arch_config_file}" ]; then mapfile -d " " -t K8S_TEST_UNION <<< "${arch_k8s_test_union}" fi -info "Run tests" -for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]} -do - bats "${K8S_TEST_ENTRY}" -done +if policy_tests_enabled; then + ensure_yq + test_actions_rejected_by_policy + add_policy_to_successful_tests +else + info "Policy tests are disabled on this platform" +fi + +test_successful_actions diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index 372ddf3a463a..1512849cbc5c 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -41,7 +41,8 @@ build_initrd() { OS_VERSION="${os_version}" \ ROOTFS_BUILD_DEST="${builddir}/initrd-image" \ USE_DOCKER=1 \ - AGENT_INIT="yes" + AGENT_INIT="yes" \ + AGENT_POLICY="${AGENT_POLICY:-}" mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}" ( cd "${install_dir}" @@ -58,7 +59,8 @@ build_image() { DEBUG="${DEBUG:-}" \ USE_DOCKER="1" \ IMG_OS_VERSION="${os_version}" \ - ROOTFS_BUILD_DEST="${builddir}/rootfs-image" + ROOTFS_BUILD_DEST="${builddir}/rootfs-image" \ + AGENT_POLICY="${AGENT_POLICY:-}" mv -f "kata-containers.img" "${install_dir}/${artifact_name}" if [ -e "root_hash.txt" ]; then cp root_hash.txt "${install_dir}/" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index a13f9c51f1b7..4a1b256f1a74 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -185,6 +185,7 @@ install_image() { #Install guest image for tdx install_image_tdx() { + export AGENT_POLICY=yes install_image "tdx" } @@ -234,11 +235,13 @@ install_initrd() { #Install Mariner guest initrd install_initrd_mariner() { + export AGENT_POLICY=yes install_initrd "mariner" } #Install guest initrd for sev install_initrd_sev() { + export AGENT_POLICY=yes install_initrd "sev" }