tests: add policy to k8s tests

Use AGENT_POLICY=yes when building the Guest images, and add a permissive test policy to the k8s tests for: - CBL-Mariner - SEV - SNP - TDX Also, add an example of policy rejecting ExecProcessRequest. Fixes: kata-containers#7667 Signed-off-by: Dan Mihai <[email protected]>
junruizh2021 · Sep 1, 2023 · bf21411 · bf21411
1 parent d0e0610
commit bf21411
Show file tree

Hide file tree

Showing 8 changed files with 177 additions and 8 deletions.
diff --git a/src/kata-opa/allow-all-except-exec-process.rego b/src/kata-opa/allow-all-except-exec-process.rego
@@ -0,0 +1,39 @@
+package agent_policy
+
+default AddARPNeighborsRequest := true
+default AddSwapRequest := true
+default CloseStdinRequest := true
+default CopyFileRequest := true
+default CreateContainerRequest := true
+default CreateSandboxRequest := true
+default DestroySandboxRequest := true
+default GetMetricsRequest := true
+default GetOOMEventRequest := true
+default GuestDetailsRequest := true
+default ListInterfacesRequest := true
+default ListRoutesRequest := true
+default MemHotplugByProbeRequest := true
+default OnlineCPUMemRequest := true
+default PauseContainerRequest := true
+default PullImageRequest := true
+default ReadStreamRequest := true
+default RemoveContainerRequest := true
+default RemoveStaleVirtiofsShareMountsRequest := true
+default ReseedRandomDevRequest := false
+default ResumeContainerRequest := true
+default SetGuestDateTimeRequest := true
+default SetPolicyRequest := true
+default SignalProcessRequest := true
+default StartContainerRequest := true
+default StartTracingRequest := true
+default StatsContainerRequest := true
+default StopTracingRequest := true
+default TtyWinResizeRequest := true
+default UpdateContainerRequest := true
+default UpdateEphemeralMountsRequest := true
+default UpdateInterfaceRequest := true
+default UpdateRoutesRequest := true
+default WaitProcessRequest := true
+default WriteStreamRequest := true
+
+default ExecProcessRequest := false
diff --git a/src/kata-opa/allow-all.rego b/src/kata-opa/allow-all.rego
@@ -7,7 +7,7 @@ default CopyFileRequest := true
 default CreateContainerRequest := true
 default CreateSandboxRequest := true
 default DestroySandboxRequest := true
-default ExecProcessRequest = true
+default ExecProcessRequest := true
 default GetMetricsRequest := true
 default GetOOMEventRequest := true
 default GuestDetailsRequest := true

diff --git a/tests/integration/kubernetes/k8s-exec-rejected.bats b/tests/integration/kubernetes/k8s-exec-rejected.bats
@@ -0,0 +1,41 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2023 Microsoft.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/../../common.bash"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+setup() {
+	get_pod_config_dir
+	pod_name="busybox"
+	pod_yaml="${pod_config_dir}/busybox-pod.yaml"
+
+	# String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all-except-exec-process.rego"
+	allow_all_except_exec_policy="cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE1ldHJpY3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RJbnRlcmZhY2VzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdFJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE1lbUhvdHBsdWdCeVByb2JlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgT25saW5lQ1BVTWVtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUGF1c2VDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQdWxsSW1hZ2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZWFkU3RyZWFtUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVzZWVkUmFuZG9tRGV2UmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IFJlc3VtZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFNldEd1ZXN0RGF0ZVRpbWVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRQb2xpY3lSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTaWduYWxQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGFydFRyYWNpbmdSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdGF0c0NvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0b3BUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVHR5V2luUmVzaXplUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlRXBoZW1lcmFsTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlSW50ZXJmYWNlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgVXBkYXRlUm91dGVzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV2FpdFByb2Nlc3NSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXcml0ZVN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQoKZGVmYXVsdCBFeGVjUHJvY2Vzc1JlcXVlc3QgOj0gZmFsc2UK"
+}
+
+@test "Kubectl exec rejected by policy" {
+	# Add to the YAML file a policy that rejects ExecProcessRequest.
+	yq write -i "${pod_yaml}" \
+		'metadata.annotations."io.katacontainers.config.agent.policy"' \
+		"${allow_all_except_exec_policy}"
+
+	# Create the pod
+	kubectl create -f "${pod_yaml}"
+
+	# Wait for pod to start
+	kubectl wait --for=condition=Ready --timeout=$timeout pod "$pod_name"
+
+	# Try executing a command in the Pod - an action rejected by the agent policy.
+	kubectl exec "$pod_name" -- date 2>&1 | grep "ExecProcessRequest is blocked by policy"
+}
+
+teardown() {
+	# Debugging information
+	kubectl describe "pod/$pod_name"
+
+	kubectl delete pod "$pod_name"
+}
diff --git a/tests/integration/kubernetes/k8s-job.bats b/tests/integration/kubernetes/k8s-job.bats
@@ -34,6 +34,11 @@ setup() {
 }
 
 teardown() {
+	# Debugging information
+	kubectl describe pod "$pod_name"
+	kubectl describe jobs/"$job_name"
+
+	# Clean-up
 	kubectl delete pod "$pod_name"
 	# Verify that pod is not running
 	run kubectl get pods

diff --git a/tests/integration/kubernetes/k8s-pod-quota.bats b/tests/integration/kubernetes/k8s-pod-quota.bats
@@ -31,6 +31,10 @@ setup() {
 }
 
 teardown() {
+	# Debugging information
+	kubectl describe deployment ${deployment_name}
+
+	# Clean-up
 	kubectl delete -f "${pod_config_dir}/pod-quota-deployment.yaml"
 	kubectl delete -f "${pod_config_dir}/resource-quota.yaml"
 }
diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh
@@ -14,6 +14,9 @@ TARGET_ARCH="${TARGET_ARCH:-x86_64}"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 K8S_TEST_DEBUG="${K8S_TEST_DEBUG:-false}"
 
+# String generated using "base64 -w 0 kata-containers/src/kata-opa/allow-all.rego"
+ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY:-"cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA9IHRydWUKZGVmYXVsdCBHZXRNZXRyaWNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR2V0T09NRXZlbnRSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBHdWVzdERldGFpbHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBMaXN0SW50ZXJmYWNlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IExpc3RSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBNZW1Ib3RwbHVnQnlQcm9iZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFBhdXNlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUHVsbEltYWdlUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVhZFN0cmVhbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlbW92ZVN0YWxlVmlydGlvZnNTaGFyZU1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlc2VlZFJhbmRvbURldlJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBSZXN1bWVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRHdWVzdERhdGVUaW1lUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0UG9saWN5UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdG9wVHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFR0eVdpblJlc2l6ZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUVwaGVtZXJhbE1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUludGVyZmFjZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZVJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdhaXRQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV3JpdGVTdHJlYW1SZXF1ZXN0IDo9IHRydWUK"}"
+
 if [ -n "${K8S_TEST_UNION:-}" ]; then
 	K8S_TEST_UNION=($K8S_TEST_UNION)
 else
@@ -58,15 +61,87 @@ else
 	)
 fi
 
+policy_tests_enabled() {
+	# The Guest images for these platforms have been built using AGENT_POLICY=yes -
+	# see kata-deploy-binaries.sh.
+	[ "${KATA_HYPERVISOR}" == "qemu-sev" ] || [ "${KATA_HYPERVISOR}" == "qemu-snp" ] || \
+		[ "${KATA_HYPERVISOR}" == "qemu-tdx" ] || [ "${KATA_HOST_OS}" == "cbl-mariner" ]
+}
+
+add_policy_to_yaml() {
+	local yaml_file="$1"
+	local resource_kind="$(yq read ${yaml_file} kind)"
+
+	case "${resource_kind}" in
+
+	Pod)
+		echo "Adding policy to ${resource_kind} from ${yaml_file}"
+		ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${K8S_TEST_YAML}" \
+			'metadata.annotations."io.katacontainers.config.agent.policy"' \
+			"${ALLOW_ALL_POLICY}"
+		;;
+
+	Deployment|Job|ReplicationController)
+		if [ "${KATA_HOST_OS}" == "cbl-mariner" ]; then
+			echo "Issue #7764: using policy for ${resource_kind} from ${yaml_file} is blocked on ${KATA_HOST_OS}"
+		else
+			echo "Adding policy to ${resource_kind} from ${yaml_file}"
+			ALLOW_ALL_POLICY="${ALLOW_ALL_POLICY}" yq write -i "${K8S_TEST_YAML}" \
+				'spec.template.metadata.annotations."io.katacontainers.config.agent.policy"' \
+				"${ALLOW_ALL_POLICY}"
+		fi
+		;;
+
+	List)
+		echo "Issue #7765: adding policy to ${resource_kind} from ${yaml_file} is not implemented yet"
+		;;
+
+	ConfigMap|LimitRange|Namespace|PersistentVolume|PersistentVolumeClaim|RuntimeClass|Secret|Service)
+		echo "Policy is not required for ${resource_kind} from ${yaml_file}"
+		;;
+
+	*)
+		echo "k8s resource type ${resource_kind} from ${yaml_file} is not yet supported for policy testing"
+		return 1
+		;;
+
+	esac
+}
+
+add_policy_to_successful_tests() {
+	info "Add policy to test YAML files"
+	for K8S_TEST_YAML in runtimeclass_workloads_work/*.yaml
+	do
+		add_policy_to_yaml "${K8S_TEST_YAML}"
+	done
+}
+
+test_successful_actions() {
+	info "Test actions that must be successful"
+	for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]}
+	do
+		bats "${K8S_TEST_ENTRY}"
+	done
+}
+
+test_actions_rejected_by_policy() {
+	info "Test k8s actions that get rejected by policy"
+	bats k8s-exec-rejected.bats
+}
+
 # we may need to skip a few test cases when running on non-x86_64 arch
 arch_config_file="${kubernetes_dir}/filter_out_per_arch/${TARGET_ARCH}.yaml"
 if [ -f "${arch_config_file}" ]; then
 	arch_k8s_test_union=$(${kubernetes_dir}/filter_k8s_test.sh ${arch_config_file} "${K8S_TEST_UNION[*]}")
 	mapfile -d " " -t K8S_TEST_UNION <<< "${arch_k8s_test_union}"
 fi
 
-info "Run tests"
-for K8S_TEST_ENTRY in ${K8S_TEST_UNION[@]}
-do
-	bats "${K8S_TEST_ENTRY}"
-done
+if policy_tests_enabled; then
+	ensure_yq
+	test_actions_rejected_by_policy
+	add_policy_to_successful_tests
+else
+	info "Policy tests are disabled on this platform"
+fi
+
+test_successful_actions
diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh
@@ -41,7 +41,8 @@ build_initrd() {
 		OS_VERSION="${os_version}" \
 		ROOTFS_BUILD_DEST="${builddir}/initrd-image" \
 		USE_DOCKER=1 \
-		AGENT_INIT="yes"
+		AGENT_INIT="yes" \
+		AGENT_POLICY="${AGENT_POLICY:-}"
 	mv "kata-containers-initrd.img" "${install_dir}/${artifact_name}"
 	(
 		cd "${install_dir}"
@@ -58,7 +59,8 @@ build_image() {
 		DEBUG="${DEBUG:-}" \
 		USE_DOCKER="1" \
 		IMG_OS_VERSION="${os_version}" \
-		ROOTFS_BUILD_DEST="${builddir}/rootfs-image"
+		ROOTFS_BUILD_DEST="${builddir}/rootfs-image" \
+		AGENT_POLICY="${AGENT_POLICY:-}"
 	mv -f "kata-containers.img" "${install_dir}/${artifact_name}"
 	if [ -e "root_hash.txt" ]; then
 	    cp root_hash.txt "${install_dir}/"

diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
@@ -185,6 +185,7 @@ install_image() {
 
 #Install guest image for tdx
 install_image_tdx() {
+	export AGENT_POLICY=yes
 	install_image "tdx"
 }
 
@@ -234,11 +235,13 @@ install_initrd() {
 
 #Install Mariner guest initrd
 install_initrd_mariner() {
+	export AGENT_POLICY=yes
 	install_initrd "mariner"
 }
 
 #Install guest initrd for sev
 install_initrd_sev() {
+	export AGENT_POLICY=yes
 	install_initrd "sev"
 }