From 93c198cebfec835208935b732f9f6d649ecfb35b Mon Sep 17 00:00:00 2001 From: Kaleemullah Siddiqui Date: Wed, 18 Mar 2026 16:59:41 +0530 Subject: [PATCH] test: Add network policy enforcement tests Add comprehensive NetworkPolicy tests for kube-apiserver and kube-apiserver-operator namespaces. Tests verify policy creation, reconciliation, and enforcement including default-deny and allow rules patterns. Signed-off-by: Kaleemullah Siddiqui --- .../main.go | 2 +- go.mod | 3 +- test/e2e/helpers.go | 828 ++++++++++++++++++ test/e2e/network_policy.go | 112 +++ test/e2e/network_policy_enforcement.go | 204 +++++ test/library/cluster_operator.go | 52 +- test/library/library.go | 2 +- 7 files changed, 1188 insertions(+), 15 deletions(-) create mode 100644 test/e2e/helpers.go create mode 100644 test/e2e/network_policy.go create mode 100644 test/e2e/network_policy_enforcement.go diff --git a/cmd/cluster-kube-apiserver-operator-tests-ext/main.go b/cmd/cluster-kube-apiserver-operator-tests-ext/main.go index f06114f23b..2947c901a2 100644 --- a/cmd/cluster-kube-apiserver-operator-tests-ext/main.go +++ b/cmd/cluster-kube-apiserver-operator-tests-ext/main.go @@ -66,7 +66,7 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { registry := oteextension.NewRegistry() extension := oteextension.NewExtension("openshift", "payload", "cluster-kube-apiserver-operator") - // The following suite runs tests that verify the operator’s behaviour. + // The following suite runs tests that verify the operator's behaviour. // This suite is executed only on pull requests targeting this repository. // Tests tagged with both [Operator] and [Serial] are included in this suite. extension.AddSuite(oteextension.Suite{ diff --git a/go.mod b/go.mod index 6eabf54ecb..663dbc0d61 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/imdario/mergo v0.3.8 github.com/miekg/dns v1.1.61 github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 github.com/openshift/api v0.0.0-20260511191110-9b69e5fa27e9 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee @@ -37,8 +38,6 @@ require ( sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 ) -require github.com/onsi/gomega v1.38.2 - require ( cel.dev/expr v0.24.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go new file mode 100644 index 0000000000..8e87d9dfda --- /dev/null +++ b/test/e2e/helpers.go @@ -0,0 +1,828 @@ +package e2e + +import ( + "context" + "fmt" + "net" + "slices" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + + test "github.com/openshift/cluster-kube-apiserver-operator/test/library" +) + +const ( + agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.45" +) + +// Client configuration helpers + +// getClientConfigForTest returns a REST config configured to connect to the api server. +// This is a wrapper around test.NewClientConfigForTest that handles errors for network policy tests. +func getClientConfigForTest() *rest.Config { + g.GinkgoHelper() + config, err := test.NewClientConfigForTest() + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get client config") + return config +} + +// createTestNamespace creates a namespace with a random suffix and returns its name. +// The namespace name is generated by appending a 5-character random string to the given prefix. +func createTestNamespace(client corev1client.NamespaceInterface, prefix string) string { + g.GinkgoHelper() + name := prefix + rand.String(5) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + _, err := client.Create(context.Background(), ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create test namespace %s", name) + return name +} + +// Network Policy validation helpers + +// getNetworkPolicy retrieves a NetworkPolicy by namespace and name. +// Fails the test if the policy cannot be retrieved. +func getNetworkPolicy(ctx context.Context, client kubernetes.Interface, namespace, name string) *networkingv1.NetworkPolicy { + g.GinkgoHelper() + policy, err := client.NetworkingV1().NetworkPolicies(namespace).Get(ctx, name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to get NetworkPolicy %s/%s", namespace, name) + return policy +} + +// requireDefaultDenyAll asserts that a NetworkPolicy is a default-deny policy: +// - empty podSelector (selects all pods in the namespace) +// - includes both Ingress and Egress policy types +// - no allow rules (empty Ingress and Egress slices) +func requireDefaultDenyAll(policy *networkingv1.NetworkPolicy) { + g.GinkgoHelper() + if len(policy.Spec.PodSelector.MatchLabels) != 0 || len(policy.Spec.PodSelector.MatchExpressions) != 0 { + g.Fail(fmt.Sprintf("%s/%s: expected empty podSelector", policy.Namespace, policy.Name)) + } + + policyTypes := sets.New[string]() + for _, policyType := range policy.Spec.PolicyTypes { + policyTypes.Insert(string(policyType)) + } + if !policyTypes.Has(string(networkingv1.PolicyTypeIngress)) || !policyTypes.Has(string(networkingv1.PolicyTypeEgress)) { + g.Fail(fmt.Sprintf("%s/%s: expected both Ingress and Egress policyTypes, got %v", policy.Namespace, policy.Name, policy.Spec.PolicyTypes)) + } + + if len(policy.Spec.Ingress) != 0 { + g.Fail(fmt.Sprintf("%s/%s: expected empty Ingress rules for default-deny policy, got %d rules", policy.Namespace, policy.Name, len(policy.Spec.Ingress))) + } + if len(policy.Spec.Egress) != 0 { + g.Fail(fmt.Sprintf("%s/%s: expected empty Egress rules for default-deny policy, got %d rules", policy.Namespace, policy.Name, len(policy.Spec.Egress))) + } +} + +// requirePodSelectorLabel asserts that a NetworkPolicy's podSelector contains a specific label. +func requirePodSelectorLabel(policy *networkingv1.NetworkPolicy, key, value string) { + g.GinkgoHelper() + actual, ok := policy.Spec.PodSelector.MatchLabels[key] + if !ok || actual != value { + g.Fail(fmt.Sprintf("%s/%s: expected podSelector %s=%s, got %v", policy.Namespace, policy.Name, key, value, policy.Spec.PodSelector.MatchLabels)) + } +} + +// requirePodSelectorExpression asserts that a NetworkPolicy's podSelector contains a matchExpression +// with the given key and values using the In operator. +func requirePodSelectorExpression(policy *networkingv1.NetworkPolicy, key string, values []string) { + g.GinkgoHelper() + found := false + for _, expr := range policy.Spec.PodSelector.MatchExpressions { + if expr.Key == key && expr.Operator == metav1.LabelSelectorOpIn { + if sets.New[string](expr.Values...).Equal(sets.New[string](values...)) { + found = true + break + } + } + } + if !found { + g.Fail(fmt.Sprintf("%s/%s: expected podSelector expression %s in %v", policy.Namespace, policy.Name, key, values)) + } +} + +// requireIngressPort asserts that a NetworkPolicy allows ingress traffic on the specified protocol and port. +func requireIngressPort(policy *networkingv1.NetworkPolicy, protocol corev1.Protocol, port int32) { + g.GinkgoHelper() + if !hasPortInIngress(policy.Spec.Ingress, protocol, port) { + g.Fail(fmt.Sprintf("%s/%s: expected ingress port %s/%d", policy.Namespace, policy.Name, protocol, port)) + } +} + +// requireIngressAllowAll asserts that a NetworkPolicy allows ingress from any source on the specified port. +func requireIngressAllowAll(policy *networkingv1.NetworkPolicy, port int32) { + g.GinkgoHelper() + if !hasIngressAllowAll(policy.Spec.Ingress, port) { + g.Fail(fmt.Sprintf("%s/%s: expected ingress allow-all on port %d", policy.Namespace, policy.Name, port)) + } +} + +// requireEgressAllowAllTCP asserts that a NetworkPolicy has an egress allow-all TCP rule. +// The rule should have no destination restrictions (empty To field) and allow TCP traffic. +func requireEgressAllowAllTCP(policy *networkingv1.NetworkPolicy) { + g.GinkgoHelper() + if !hasEgressAllowAllTCP(policy.Spec.Egress) { + o.Expect(hasEgressAllowAllTCP(policy.Spec.Egress)).To(o.BeTrue(), + fmt.Sprintf("NetworkPolicy %s/%s should have egress allow-all TCP rule", policy.Namespace, policy.Name)) + } +} + +func hasIngressAllowAll(rules []networkingv1.NetworkPolicyIngressRule, port int32) bool { + for _, rule := range rules { + if !hasPort(rule.Ports, corev1.ProtocolTCP, port) { + continue + } + if len(rule.From) == 0 { + return true + } + } + return false +} + +func hasEgressAllowAllTCP(rules []networkingv1.NetworkPolicyEgressRule) bool { + for _, rule := range rules { + if len(rule.To) != 0 { + continue + } + if hasAnyTCPPort(rule.Ports) { + return true + } + } + return false +} + +func hasAnyTCPPort(ports []networkingv1.NetworkPolicyPort) bool { + if len(ports) == 0 { + return true + } + for _, p := range ports { + if p.Protocol != nil && *p.Protocol != corev1.ProtocolTCP { + continue + } + return true + } + return false +} + +func hasPortInIngress(rules []networkingv1.NetworkPolicyIngressRule, protocol corev1.Protocol, port int32) bool { + for _, rule := range rules { + if hasPort(rule.Ports, protocol, port) { + return true + } + } + return false +} + +func hasPort(ports []networkingv1.NetworkPolicyPort, protocol corev1.Protocol, port int32) bool { + for _, p := range ports { + if p.Port == nil || p.Port.IntValue() != int(port) { + continue + } + if p.Protocol == nil || *p.Protocol == protocol { + return true + } + } + return false +} + +// deleteAndWaitForAllRestored deletes all given NetworkPolicies at once, then +// polls until every one is restored by the operator. This avoids the sequential +// 10-minute-per-policy timeout that made the test slow on certain platforms. +func deleteAndWaitForAllRestored(ctx context.Context, client kubernetes.Interface, policies []*networkingv1.NetworkPolicy) { + g.GinkgoHelper() + + type policyState struct { + expected *networkingv1.NetworkPolicy + originalUID types.UID + sawDeletion bool + confirmed bool + } + states := make([]policyState, len(policies)) + + for i, p := range policies { + states[i] = policyState{expected: p, originalUID: p.UID} + g.GinkgoWriter.Printf("deleting NetworkPolicy %s/%s (UID=%s)\n", p.Namespace, p.Name, p.UID) + err := client.NetworkingV1().NetworkPolicies(p.Namespace).Delete(ctx, p.Name, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to delete NetworkPolicy %s/%s", p.Namespace, p.Name) + } + + err := wait.PollUntilContextTimeout(ctx, 15*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + allRestored := true + for i := range states { + if states[i].confirmed { + continue + } + ns := states[i].expected.Namespace + name := states[i].expected.Name + current, err := client.NetworkingV1().NetworkPolicies(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + states[i].sawDeletion = true + allRestored = false + continue + } + if err != nil { + allRestored = false + continue + } + if current.UID == states[i].originalUID && !states[i].sawDeletion { + allRestored = false + continue + } + if !equality.Semantic.DeepEqual(states[i].expected.Spec, current.Spec) { + allRestored = false + continue + } + g.GinkgoWriter.Printf("NetworkPolicy %s/%s restored\n", ns, name) + states[i].confirmed = true + } + return allRestored, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "timed out waiting for all NetworkPolicies to be restored after deletion") +} + +// mutateAndWaitForAllReconciled mutates all given NetworkPolicies at once, then +// polls until the operator reconciles every one back to its original spec. +func mutateAndWaitForAllReconciled(ctx context.Context, client kubernetes.Interface, policies []struct{ namespace, name string }) { + g.GinkgoHelper() + + type mutationState struct { + namespace string + name string + original networkingv1.NetworkPolicySpec + confirmed bool + } + states := make([]mutationState, 0, len(policies)) + + patch := []byte(`{"spec":{"podSelector":{"matchLabels":{"np-reconcile":"mutated"}}}}`) + for _, p := range policies { + original := getNetworkPolicy(ctx, client, p.namespace, p.name) + g.GinkgoWriter.Printf("mutating NetworkPolicy %s/%s\n", p.namespace, p.name) + _, err := client.NetworkingV1().NetworkPolicies(p.namespace).Patch(ctx, p.name, types.MergePatchType, patch, metav1.PatchOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to patch NetworkPolicy %s/%s", p.namespace, p.name) + states = append(states, mutationState{namespace: p.namespace, name: p.name, original: original.Spec}) + } + + time.Sleep(2 * time.Second) + + err := wait.PollUntilContextTimeout(ctx, 15*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + allReconciled := true + for i := range states { + s := &states[i] + if s.confirmed { + continue + } + current, err := client.NetworkingV1().NetworkPolicies(s.namespace).Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + g.GinkgoWriter.Printf("waiting for NetworkPolicy %s/%s reconciliation: %v\n", s.namespace, s.name, err) + allReconciled = false + continue + } + // For fast-reconciling operators, the mutation may be reverted before we can observe it. + // We trust that the patch succeeded and just verify the final state matches the original. + if !equality.Semantic.DeepEqual(s.original, current.Spec) { + allReconciled = false + continue + } + g.GinkgoWriter.Printf("NetworkPolicy %s/%s reconciled\n", s.namespace, s.name) + s.confirmed = true + } + return allReconciled, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "timed out waiting for all NetworkPolicies to be reconciled after mutation") +} + +func waitForPodsReadyByLabel(ctx context.Context, client kubernetes.Interface, namespace, labelSelector string) { + g.GinkgoHelper() + g.GinkgoWriter.Printf("waiting for pods ready in %s with selector %s\n", namespace, labelSelector) + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return false, err + } + if len(pods.Items) == 0 { + return false, nil + } + for _, pod := range pods.Items { + if !isPodReady(&pod) { + return false, nil + } + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "timed out waiting for pods in %s with selector %s to be ready", namespace, labelSelector) +} + +func isPodReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func logNetworkPolicyEvents(ctx context.Context, client kubernetes.Interface, namespaces []string, policyName string) { + g.GinkgoHelper() + found := false + _ = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + for _, namespace := range namespaces { + events, err := client.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + g.GinkgoWriter.Printf("unable to list events in %s: %v\n", namespace, err) + continue + } + for _, event := range events.Items { + if (event.InvolvedObject.Kind == "NetworkPolicy" && event.InvolvedObject.Name == policyName) || + (event.Message != "" && (event.InvolvedObject.Name == policyName || event.InvolvedObject.Kind == "NetworkPolicy")) { + g.GinkgoWriter.Printf("event in %s: %s %s %s\n", namespace, event.Type, event.Reason, event.Message) + found = true + } + } + } + if found { + return true, nil + } + g.GinkgoWriter.Printf("no NetworkPolicy events yet for %s (namespaces: %v)\n", policyName, namespaces) + return false, nil + }) + if !found { + g.GinkgoWriter.Printf("no NetworkPolicy events observed for %s (best-effort)\n", policyName) + } +} + +func logNetworkPolicySummary(label string, policy *networkingv1.NetworkPolicy) { + g.GinkgoWriter.Printf("networkpolicy %s namespace=%s name=%s podSelector=%v policyTypes=%v ingress=%d egress=%d\n", + label, + policy.Namespace, + policy.Name, + policy.Spec.PodSelector.MatchLabels, + policy.Spec.PolicyTypes, + len(policy.Spec.Ingress), + len(policy.Spec.Egress), + ) +} + +func logNetworkPolicyDetails(label string, policy *networkingv1.NetworkPolicy) { + g.GinkgoHelper() + g.GinkgoWriter.Printf("networkpolicy %s details:\n", label) + g.GinkgoWriter.Printf(" podSelector=%v policyTypes=%v\n", policy.Spec.PodSelector.MatchLabels, policy.Spec.PolicyTypes) + for i, rule := range policy.Spec.Ingress { + g.GinkgoWriter.Printf(" ingress[%d]: ports=%s from=%s\n", i, formatPorts(rule.Ports), formatPeers(rule.From)) + } + for i, rule := range policy.Spec.Egress { + g.GinkgoWriter.Printf(" egress[%d]: ports=%s to=%s\n", i, formatPorts(rule.Ports), formatPeers(rule.To)) + } +} + +func formatPorts(ports []networkingv1.NetworkPolicyPort) string { + if len(ports) == 0 { + return "[]" + } + out := make([]string, 0, len(ports)) + for _, p := range ports { + proto := "TCP" + if p.Protocol != nil { + proto = string(*p.Protocol) + } + if p.Port == nil { + out = append(out, fmt.Sprintf("%s:any", proto)) + continue + } + out = append(out, fmt.Sprintf("%s:%s", proto, p.Port.String())) + } + return fmt.Sprintf("[%s]", strings.Join(out, ", ")) +} + +func formatPeers(peers []networkingv1.NetworkPolicyPeer) string { + if len(peers) == 0 { + return "[]" + } + out := make([]string, 0, len(peers)) + for _, peer := range peers { + ns := formatSelector(peer.NamespaceSelector) + pod := formatSelector(peer.PodSelector) + if ns == "" && pod == "" { + out = append(out, "{}") + continue + } + out = append(out, fmt.Sprintf("ns=%s pod=%s", ns, pod)) + } + return fmt.Sprintf("[%s]", strings.Join(out, ", ")) +} + +func formatSelector(sel *metav1.LabelSelector) string { + if sel == nil { + return "" + } + if len(sel.MatchLabels) == 0 && len(sel.MatchExpressions) == 0 { + return "{}" + } + return fmt.Sprintf("labels=%v exprs=%v", sel.MatchLabels, sel.MatchExpressions) +} + +// Network Policy enforcement helpers + +// ensureTestServiceAccount creates a service account for network policy tests if it doesn't exist. +// Returns the name of the service account. +func ensureTestServiceAccount(ctx context.Context, kubeClient kubernetes.Interface, namespace string) string { + g.GinkgoHelper() + saName := "netpolicy-test-sa" + + // Check if service account already exists + _, err := kubeClient.CoreV1().ServiceAccounts(namespace).Get(ctx, saName, metav1.GetOptions{}) + if err == nil { + // Service account already exists + return saName + } + + if !apierrors.IsNotFound(err) { + o.Expect(err).NotTo(o.HaveOccurred(), "failed to check for existing service account") + } + + // Create the service account + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: namespace, + }, + } + + _, err = kubeClient.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred(), "failed to create test service account") + + g.GinkgoWriter.Printf("created test service account %s/%s\n", namespace, saName) + return saName +} + +func netexecPod(name, namespace string, labels map[string]string, port int32) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: map[string]string{ + "openshift.io/required-scc": "nonroot-v2", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "netpolicy-test-sa", + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{ + { + Name: "netexec", + Image: agnhostImage, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + }, + Command: []string{"/agnhost"}, + Args: []string{"netexec", fmt.Sprintf("--http-port=%d", port)}, + Ports: []corev1.ContainerPort{ + {ContainerPort: port}, + }, + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + }, + }, + }, + } +} + +func createServerPod(ctx context.Context, kubeClient kubernetes.Interface, namespace, name string, labels map[string]string, port int32) ([]string, func()) { + g.GinkgoHelper() + + g.GinkgoWriter.Printf("creating server pod %s/%s port=%d labels=%v\n", namespace, name, port, labels) + pod := netexecPod(name, namespace, labels, port) + _, err := kubeClient.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(waitForPodReady(ctx, kubeClient, namespace, name)).NotTo(o.HaveOccurred()) + + created, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(created.Status.PodIPs).NotTo(o.BeEmpty()) + + ips := podIPs(created) + g.GinkgoWriter.Printf("server pod %s/%s ips=%v\n", namespace, name, ips) + + return ips, func() { + g.GinkgoWriter.Printf("deleting server pod %s/%s\n", namespace, name) + _ = kubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + } +} + +func podIPs(pod *corev1.Pod) []string { + var ips []string + for _, podIP := range pod.Status.PodIPs { + if podIP.IP != "" { + ips = append(ips, podIP.IP) + } + } + if len(ips) == 0 && pod.Status.PodIP != "" { + ips = append(ips, pod.Status.PodIP) + } + return ips +} + +func expectConnectivityForIP(ctx context.Context, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIP string, port int32, shouldSucceed bool) { + g.GinkgoHelper() + + podName, cleanup, err := createConnectivityClientPod(ctx, kubeClient, namespace, clientLabels, serverIP, port) + o.Expect(err).NotTo(o.HaveOccurred()) + g.DeferCleanup(cleanup) + + err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + succeeded, err := readConnectivityResult(ctx, kubeClient, namespace, podName) + if err != nil { + g.GinkgoWriter.Printf("waiting for connectivity result: %v\n", err) + return false, nil + } + return succeeded == shouldSucceed, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + g.GinkgoWriter.Printf("connectivity %s/%s expected=%t\n", namespace, formatIPPort(serverIP, port), shouldSucceed) +} + +func expectConnectivity(ctx context.Context, kubeClient kubernetes.Interface, namespace string, clientLabels map[string]string, serverIPs []string, port int32, shouldSucceed bool) { + g.GinkgoHelper() + + for _, ip := range serverIPs { + family := "IPv4" + if isIPv6(ip) { + family = "IPv6" + } + g.GinkgoWriter.Printf("checking %s connectivity %s -> %s expected=%t\n", family, namespace, formatIPPort(ip, port), shouldSucceed) + expectConnectivityForIP(ctx, kubeClient, namespace, clientLabels, ip, port, shouldSucceed) + } +} + +func createConnectivityClientPod(ctx context.Context, kubeClient kubernetes.Interface, namespace string, labels map[string]string, serverIP string, port int32) (string, func(), error) { + name := fmt.Sprintf("np-client-%s", rand.String(5)) + target := formatIPPort(serverIP, port) + + g.GinkgoWriter.Printf("creating client pod %s/%s to probe %s\n", namespace, name, target) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: map[string]string{ + "openshift.io/required-scc": "nonroot-v2", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "netpolicy-test-sa", + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{ + { + Name: "connect", + Image: agnhostImage, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: boolptr(true), + RunAsUser: int64ptr(1001), + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{ + fmt.Sprintf("while true; do if /agnhost connect --protocol=tcp --timeout=5s %s 2>/dev/null; then echo CONN_OK; else echo CONN_FAIL; fi; sleep 3; done", target), + }, + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + }, + }, + }, + } + + _, err := kubeClient.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return "", nil, err + } + + if err := waitForPodReady(ctx, kubeClient, namespace, name); err != nil { + _ = kubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + return "", nil, fmt.Errorf("client pod %s/%s never became ready: %w", namespace, name, err) + } + + cleanup := func() { + g.GinkgoWriter.Printf("deleting client pod %s/%s\n", namespace, name) + _ = kubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + } + + return name, cleanup, nil +} + +func readConnectivityResult(ctx context.Context, kubeClient kubernetes.Interface, namespace, podName string) (bool, error) { + tailLines := int64(1) + raw, err := kubeClient.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{ + TailLines: &tailLines, + }).DoRaw(ctx) + if err != nil { + return false, err + } + + line := strings.TrimSpace(string(raw)) + if line == "" { + return false, fmt.Errorf("no connectivity result yet from pod %s/%s", namespace, podName) + } + + g.GinkgoWriter.Printf("client pod %s/%s result=%s\n", namespace, podName, line) + return line == "CONN_OK", nil +} + +func ingressAllowsFromNamespace(policy *networkingv1.NetworkPolicy, namespace string, labels map[string]string, port int32) bool { + for _, rule := range policy.Spec.Ingress { + if !ruleAllowsPort(rule.Ports, port) { + continue + } + if len(rule.From) == 0 { + return true + } + for _, peer := range rule.From { + if peer.NamespaceSelector != nil { + if nsMatch(peer.NamespaceSelector, namespace) && podMatch(peer.PodSelector, labels) { + return true + } + continue + } + if podMatch(peer.PodSelector, labels) { + return true + } + } + } + return false +} + +func nsMatch(selector *metav1.LabelSelector, namespace string) bool { + if selector == nil { + return true + } + if selector.MatchLabels != nil { + if selector.MatchLabels["kubernetes.io/metadata.name"] == namespace { + return true + } + } + for _, expr := range selector.MatchExpressions { + if expr.Key != "kubernetes.io/metadata.name" { + continue + } + if expr.Operator != metav1.LabelSelectorOpIn { + continue + } + if slices.Contains(expr.Values, namespace) { + return true + } + } + return false +} + +func podMatch(selector *metav1.LabelSelector, labels map[string]string) bool { + if selector == nil { + return true + } + for key, value := range selector.MatchLabels { + if labels[key] != value { + return false + } + } + return true +} + +func ruleAllowsPort(ports []networkingv1.NetworkPolicyPort, port int32) bool { + if len(ports) == 0 { + return true + } + for _, p := range ports { + if p.Port == nil { + return true + } + if p.Port.Type == intstr.Int && p.Port.IntVal == port { + return true + } + } + return false +} + +func waitForPodReady(ctx context.Context, kubeClient kubernetes.Interface, namespace, name string) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + pod, err := kubeClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if pod.Status.Phase != corev1.PodRunning { + return false, nil + } + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) +} + +// NetworkPolicy construction helpers + +func defaultDenyPolicy(name, namespace string) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress}, + }, + } +} + +func allowIngressPolicy(name, namespace string, podLabels, fromLabels map[string]string, port int32) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: podLabels}, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: fromLabels}}, + }, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: port}, Protocol: protocolPtr(corev1.ProtocolTCP)}, + }, + }, + }, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + }, + } +} + +func allowEgressPolicy(name, namespace string, podLabels, toLabels map[string]string, port int32) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: podLabels}, + Egress: []networkingv1.NetworkPolicyEgressRule{ + { + To: []networkingv1.NetworkPolicyPeer{ + {PodSelector: &metav1.LabelSelector{MatchLabels: toLabels}}, + }, + Ports: []networkingv1.NetworkPolicyPort{ + {Port: &intstr.IntOrString{Type: intstr.Int, IntVal: port}, Protocol: protocolPtr(corev1.ProtocolTCP)}, + }, + }, + }, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + }, + } +} + +// Utility helpers + +func isIPv6(ip string) bool { + return net.ParseIP(ip) != nil && strings.Contains(ip, ":") +} + +func formatIPPort(ip string, port int32) string { + if isIPv6(ip) { + return fmt.Sprintf("[%s]:%d", ip, port) + } + return fmt.Sprintf("%s:%d", ip, port) +} + +func protocolPtr(protocol corev1.Protocol) *corev1.Protocol { + return &protocol +} + +func boolptr(value bool) *bool { + return &value +} + +func int64ptr(value int64) *int64 { + return &value +} diff --git a/test/e2e/network_policy.go b/test/e2e/network_policy.go new file mode 100644 index 0000000000..d4123882cf --- /dev/null +++ b/test/e2e/network_policy.go @@ -0,0 +1,112 @@ +package e2e + +import ( + "context" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/client-go/kubernetes" + + configclient "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + test "github.com/openshift/cluster-kube-apiserver-operator/test/library" +) + +const ( + kubeAPIServerNamespace = "openshift-kube-apiserver" + kubeAPIServerOperatorNamespace = "openshift-kube-apiserver-operator" + defaultDenyAllPolicyName = "default-deny" + operatorAllowPolicyName = "allow-all-egress-and-metrics-ingress" + operandAllowPolicyName = "allow-all-egress" +) + +var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { + g.It("[Operator][NetworkPolicy] should ensure kube-apiserver NetworkPolicies are defined", func() { + testKubeAPIServerNetworkPolicies() + }) + g.It("[Serial][Operator][NetworkPolicy] should restore kube-apiserver NetworkPolicies after delete or mutation", func() { + testKubeAPIServerNetworkPolicyReconcile() + }) +}) + +func testKubeAPIServerNetworkPolicies() { + ctx := context.Background() + g.By("Creating Kubernetes clients") + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + configClient, err := configclient.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for kube-apiserver ClusterOperator to be stable") + err = test.WaitForClusterOperatorAvailableNotProgressingNotDegraded(g.GinkgoTB(), configClient, "kube-apiserver") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validating NetworkPolicies in openshift-kube-apiserver-operator") + operatorDefaultDeny := getNetworkPolicy(ctx, kubeClient, kubeAPIServerOperatorNamespace, defaultDenyAllPolicyName) + logNetworkPolicySummary("kube-apiserver-operator/default-deny-all", operatorDefaultDeny) + logNetworkPolicyDetails("kube-apiserver-operator/default-deny-all", operatorDefaultDeny) + requireDefaultDenyAll(operatorDefaultDeny) + + operatorAllowPolicy := getNetworkPolicy(ctx, kubeClient, kubeAPIServerOperatorNamespace, operatorAllowPolicyName) + logNetworkPolicySummary("kube-apiserver-operator/"+operatorAllowPolicyName, operatorAllowPolicy) + logNetworkPolicyDetails("kube-apiserver-operator/"+operatorAllowPolicyName, operatorAllowPolicy) + requirePodSelectorLabel(operatorAllowPolicy, "app", "kube-apiserver-operator") + requireIngressPort(operatorAllowPolicy, corev1.ProtocolTCP, 8443) + requireIngressAllowAll(operatorAllowPolicy, 8443) + requireEgressAllowAllTCP(operatorAllowPolicy) + + g.By("Validating NetworkPolicies in openshift-kube-apiserver") + operandDefaultDeny := getNetworkPolicy(ctx, kubeClient, kubeAPIServerNamespace, defaultDenyAllPolicyName) + logNetworkPolicySummary("kube-apiserver/default-deny-all", operandDefaultDeny) + logNetworkPolicyDetails("kube-apiserver/default-deny-all", operandDefaultDeny) + requireDefaultDenyAll(operandDefaultDeny) + + operandAllowPolicy := getNetworkPolicy(ctx, kubeClient, kubeAPIServerNamespace, operandAllowPolicyName) + logNetworkPolicySummary("kube-apiserver/"+operandAllowPolicyName, operandAllowPolicy) + logNetworkPolicyDetails("kube-apiserver/"+operandAllowPolicyName, operandAllowPolicy) + // Verify it selects guard, installer, and pruner pods with In expression + requirePodSelectorExpression(operandAllowPolicy, "app", []string{"guard", "installer", "pruner"}) + requireEgressAllowAllTCP(operandAllowPolicy) + + g.By("Verifying pods are ready in kube-apiserver namespaces") + waitForPodsReadyByLabel(ctx, kubeClient, kubeAPIServerOperatorNamespace, "app=kube-apiserver-operator") +} + +func testKubeAPIServerNetworkPolicyReconcile() { + ctx := context.Background() + g.By("Creating Kubernetes clients") + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Capturing expected NetworkPolicy specs across all namespaces") + expectedOperatorPolicy := getNetworkPolicy(ctx, kubeClient, kubeAPIServerOperatorNamespace, operatorAllowPolicyName) + expectedOperandPolicy := getNetworkPolicy(ctx, kubeClient, kubeAPIServerNamespace, operandAllowPolicyName) + expectedOperatorDefaultDeny := getNetworkPolicy(ctx, kubeClient, kubeAPIServerOperatorNamespace, defaultDenyAllPolicyName) + expectedOperandDefaultDeny := getNetworkPolicy(ctx, kubeClient, kubeAPIServerNamespace, defaultDenyAllPolicyName) + + policiesToDelete := []*networkingv1.NetworkPolicy{ + expectedOperatorPolicy, + expectedOperandPolicy, + expectedOperatorDefaultDeny, + expectedOperandDefaultDeny, + } + + g.By("Deleting all NetworkPolicies simultaneously and waiting for restoration") + deleteAndWaitForAllRestored(ctx, kubeClient, policiesToDelete) + + g.By("Mutating all NetworkPolicies simultaneously and waiting for reconciliation") + policiesToMutate := []struct{ namespace, name string }{ + {kubeAPIServerOperatorNamespace, operatorAllowPolicyName}, + {kubeAPIServerNamespace, operandAllowPolicyName}, + {kubeAPIServerOperatorNamespace, defaultDenyAllPolicyName}, + {kubeAPIServerNamespace, defaultDenyAllPolicyName}, + } + mutateAndWaitForAllReconciled(ctx, kubeClient, policiesToMutate) + + g.By("Checking NetworkPolicy-related events (best-effort)") + logNetworkPolicyEvents(ctx, kubeClient, []string{kubeAPIServerOperatorNamespace, kubeAPIServerNamespace}, operatorAllowPolicyName) +} diff --git a/test/e2e/network_policy_enforcement.go b/test/e2e/network_policy_enforcement.go new file mode 100644 index 0000000000..c4b6d08fe0 --- /dev/null +++ b/test/e2e/network_policy_enforcement.go @@ -0,0 +1,204 @@ +package e2e + +import ( + "context" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { + g.It("[Operator][NetworkPolicy] should enforce NetworkPolicy allow/deny basics in a test namespace", func() { + testGenericNetworkPolicyEnforcement() + }) + g.It("[Operator][NetworkPolicy] should enforce kube-apiserver-operator NetworkPolicies", func() { + testKubeAPIServerOperatorNetworkPolicyEnforcement() + }) + g.It("[Operator][NetworkPolicy] should enforce cross-namespace ingress traffic", func() { + testCrossNamespaceIngressEnforcement() + }) + g.It("[Operator][NetworkPolicy] should allow metrics but block other ports", func() { + testMetricsOpenButOtherPortsBlocked() + }) + g.It("[Operator][NetworkPolicy] should allow metrics ingress from allowed namespaces only", func() { + testMetricsIngressOpenAccess() + }) +}) + +func testGenericNetworkPolicyEnforcement() { + ctx := context.Background() + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating a temporary namespace for policy enforcement checks") + nsName := createTestNamespace(kubeClient.CoreV1().Namespaces(), "np-enforcement-") + g.DeferCleanup(func() { + g.GinkgoWriter.Printf("deleting test namespace %s\n", nsName) + _ = kubeClient.CoreV1().Namespaces().Delete(ctx, nsName, metav1.DeleteOptions{}) + }) + + g.By("Creating test service account") + ensureTestServiceAccount(ctx, kubeClient, nsName) + + serverName := "np-server" + clientLabels := map[string]string{"app": "np-client"} + serverLabels := map[string]string{"app": "np-server"} + + g.GinkgoWriter.Printf("creating netexec server pod %s/%s\n", nsName, serverName) + serverPod := netexecPod(serverName, nsName, serverLabels, 8080) + _, err = kubeClient.CoreV1().Pods(nsName).Create(ctx, serverPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(waitForPodReady(ctx, kubeClient, nsName, serverName)).NotTo(o.HaveOccurred()) + + server, err := kubeClient.CoreV1().Pods(nsName).Get(ctx, serverName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(server.Status.PodIPs).NotTo(o.BeEmpty()) + serverIPs := podIPs(server) + g.GinkgoWriter.Printf("server pod %s/%s ips=%v\n", nsName, serverName, serverIPs) + + g.By("Verifying allow-all when no policies select the pod") + expectConnectivity(ctx, kubeClient, nsName, clientLabels, serverIPs, 8080, true) + + g.By("Applying default deny and verifying traffic is blocked") + g.GinkgoWriter.Printf("creating default-deny policy in %s\n", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(ctx, defaultDenyPolicy("default-deny", nsName), metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Adding ingress allow only and verifying traffic is still blocked") + g.GinkgoWriter.Printf("creating allow-ingress policy in %s\n", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(ctx, allowIngressPolicy("allow-ingress", nsName, serverLabels, clientLabels, 8080), metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + expectConnectivity(ctx, kubeClient, nsName, clientLabels, serverIPs, 8080, false) + + g.By("Adding egress allow and verifying traffic is permitted") + g.GinkgoWriter.Printf("creating allow-egress policy in %s\n", nsName) + _, err = kubeClient.NetworkingV1().NetworkPolicies(nsName).Create(ctx, allowEgressPolicy("allow-egress", nsName, clientLabels, serverLabels, 8080), metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + expectConnectivity(ctx, kubeClient, nsName, clientLabels, serverIPs, 8080, true) +} + +func testKubeAPIServerOperatorNetworkPolicyEnforcement() { + ctx := context.Background() + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + namespace := "openshift-kube-apiserver-operator" + + g.By("Creating test service account in monitoring namespace for connectivity tests") + ensureTestServiceAccount(ctx, kubeClient, "openshift-monitoring") + + g.By("Getting IP of real kube-apiserver-operator pod") + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=kube-apiserver-operator", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "should have at least one operator pod") + + operatorPodIPs := podIPs(&pods.Items[0]) + g.GinkgoWriter.Printf("testing network policy enforcement using real operator pod IPs: %v\n", operatorPodIPs) + + g.By("Verifying cross-namespace traffic from monitoring to operator metrics port is allowed") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, operatorPodIPs, 8443, true) + + g.By("Verifying unauthorized ports are denied by default-deny policy") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, operatorPodIPs, 12345, false) +} + +func testCrossNamespaceIngressEnforcement() { + ctx := context.Background() + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating test service account in monitoring namespace") + ensureTestServiceAccount(ctx, kubeClient, "openshift-monitoring") + + g.By("Getting IP of real kube-apiserver-operator pod") + pods, err := kubeClient.CoreV1().Pods("openshift-kube-apiserver-operator").List(ctx, metav1.ListOptions{ + LabelSelector: "app=kube-apiserver-operator", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "should have at least one operator pod") + kasOperatorIPs := podIPs(&pods.Items[0]) + + g.By("Testing cross-namespace ingress: monitoring -> kube-apiserver-operator:8443") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, kasOperatorIPs, 8443, true) + + g.By("Testing cross-namespace ingress: any pod from monitoring can access metrics") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app": "any-label"}, kasOperatorIPs, 8443, true) +} + +func testMetricsOpenButOtherPortsBlocked() { + ctx := context.Background() + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating test service accounts in required namespaces") + ensureTestServiceAccount(ctx, kubeClient, "default") + ensureTestServiceAccount(ctx, kubeClient, "openshift-etcd") + ensureTestServiceAccount(ctx, kubeClient, "openshift-monitoring") + + g.By("Getting IP of real kube-apiserver-operator pod") + pods, err := kubeClient.CoreV1().Pods("openshift-kube-apiserver-operator").List(ctx, metav1.ListOptions{ + LabelSelector: "app=kube-apiserver-operator", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "should have at least one operator pod") + kasOperatorIPs := podIPs(&pods.Items[0]) + + g.By("Testing metrics port 8443 is now open: default namespace -> kube-apiserver-operator:8443") + expectConnectivity(ctx, kubeClient, "default", map[string]string{"test": "client"}, kasOperatorIPs, 8443, true) + + g.By("Testing metrics port 8443 from openshift-etcd with custom app label: should be denied") + expectConnectivity(ctx, kubeClient, "openshift-etcd", map[string]string{"test": "client"}, kasOperatorIPs, 8443, false) + + g.By("Testing port-based blocking: unauthorized ports are still blocked") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, kasOperatorIPs, 9999, false) + + g.By("Testing multiple unauthorized ports are still blocked by default-deny") + for _, port := range []int32{80, 443, 8080, 22, 3306, 9090} { + expectConnectivity(ctx, kubeClient, "default", map[string]string{"test": "any-pod"}, kasOperatorIPs, port, false) + } +} + +func testMetricsIngressOpenAccess() { + ctx := context.Background() + kubeConfig := getClientConfigForTest() + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating test service accounts in required namespaces") + ensureTestServiceAccount(ctx, kubeClient, "openshift-monitoring") + ensureTestServiceAccount(ctx, kubeClient, "openshift-etcd") + ensureTestServiceAccount(ctx, kubeClient, "openshift-console") + ensureTestServiceAccount(ctx, kubeClient, "default") + + g.By("Getting IP of real kube-apiserver-operator pod") + pods, err := kubeClient.CoreV1().Pods("openshift-kube-apiserver-operator").List(ctx, metav1.ListOptions{ + LabelSelector: "app=kube-apiserver-operator", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(pods.Items).NotTo(o.BeEmpty(), "should have at least one operator pod") + kasOperatorIPs := podIPs(&pods.Items[0]) + + g.By("Testing allow-to-metrics policy: monitoring namespace can access metrics -> operator:8443") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, kasOperatorIPs, 8443, true) + + g.By("Testing metrics policy: etcd namespace with custom app label should be denied") + expectConnectivity(ctx, kubeClient, "openshift-etcd", map[string]string{"test": "metrics-client"}, kasOperatorIPs, 8443, false) + + g.By("Testing metrics policy: console namespace with custom app label can access metrics") + expectConnectivity(ctx, kubeClient, "openshift-console", map[string]string{"custom-app": "test-client"}, kasOperatorIPs, 8443, true) + + g.By("Testing allow-to-metrics policy: default namespace can access metrics -> operator:8443") + expectConnectivity(ctx, kubeClient, "default", map[string]string{"test": "client"}, kasOperatorIPs, 8443, true) + + g.By("Testing default-deny still blocks unauthorized ports") + expectConnectivity(ctx, kubeClient, "openshift-monitoring", map[string]string{"app.kubernetes.io/name": "prometheus"}, kasOperatorIPs, 9090, false) +} diff --git a/test/library/cluster_operator.go b/test/library/cluster_operator.go index 4d16d64ff8..24b12b2c75 100644 --- a/test/library/cluster_operator.go +++ b/test/library/cluster_operator.go @@ -14,18 +14,46 @@ import ( clusteroperatorhelpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" ) +// WaitForClusterOperatorAvailableNotProgressingNotDegraded waits for a ClusterOperator to report +// status as Available=true, Progressing=false, and Degraded=false. +// Returns an error if the wait times out or encounters an error while checking the ClusterOperator status. +func WaitForClusterOperatorAvailableNotProgressingNotDegraded(t testing.TB, client configclient.ConfigV1Interface, name string) error { + ctx := context.Background() + err := wait.PollUntilContextTimeout(ctx, WaitPollInterval, WaitPollTimeout, true, func(ctx context.Context) (bool, error) { + clusterOperator, err := client.ClusterOperators().Get(ctx, name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + fmt.Printf("ClusterOperator/%s does not yet exist.\n", name) + return false, nil + } + if err != nil { + fmt.Printf("Unable to retrieve ClusterOperator/%s (retrying): %v\n", name, err) + return false, nil + } + conditions := clusterOperator.Status.Conditions + available := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorAvailable, configv1.ConditionTrue) + notProgressing := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorProgressing, configv1.ConditionFalse) + notDegraded := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorDegraded, configv1.ConditionFalse) + done := available && notProgressing && notDegraded + fmt.Printf("ClusterOperator/%s: Available: %v Progressing: %v Degraded: %v\n", name, available, !notProgressing, !notDegraded) + return done, nil + }) + return err +} + // WaitForKubeAPIServerClusterOperatorAvailableNotProgressingNotDegraded waits for ClusterOperator/kube-apiserver to report -// status as active, not progressing, and not failing. +// status as Available=true, Progressing=false, and Degraded=false. +// Calls t.Fatal if the wait times out or encounters an error. func WaitForKubeAPIServerClusterOperatorAvailableNotProgressingNotDegraded(t *testing.T, client configclient.ConfigV1Interface) { - err := wait.Poll(WaitPollInterval, WaitPollTimeout, func() (bool, error) { - clusterOperator, err := client.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + ctx := context.Background() + err := wait.PollUntilContextTimeout(ctx, WaitPollInterval, WaitPollTimeout, true, func(ctx context.Context) (bool, error) { + clusterOperator, err := client.ClusterOperators().Get(ctx, "kube-apiserver", metav1.GetOptions{}) if errors.IsNotFound(err) { fmt.Println("ClusterOperator/kube-apiserver does not yet exist.") return false, nil } if err != nil { - fmt.Println("Unable to retrieve ClusterOperator/kube-apiserver:", err) - return false, err + fmt.Println("Unable to retrieve ClusterOperator/kube-apiserver (retrying):", err) + return false, nil } conditions := clusterOperator.Status.Conditions available := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorAvailable, configv1.ConditionTrue) @@ -40,18 +68,20 @@ func WaitForKubeAPIServerClusterOperatorAvailableNotProgressingNotDegraded(t *te } } -// WaitForKubeAPIServer waits for ClusterOperator/kube-apiserver to report -// status as active, progressing, and not failing. +// WaitForKubeAPIServerStartProgressing waits for ClusterOperator/kube-apiserver to report +// status as Available=true, Progressing=true, and Degraded=false. +// Calls t.Fatal if the wait times out or encounters an error. func WaitForKubeAPIServerStartProgressing(t *testing.T, client configclient.ConfigV1Interface) { - err := wait.Poll(WaitPollInterval, WaitPollTimeout, func() (bool, error) { - clusterOperator, err := client.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + ctx := context.Background() + err := wait.PollUntilContextTimeout(ctx, WaitPollInterval, WaitPollTimeout, true, func(ctx context.Context) (bool, error) { + clusterOperator, err := client.ClusterOperators().Get(ctx, "kube-apiserver", metav1.GetOptions{}) if errors.IsNotFound(err) { fmt.Println("ClusterOperator/kube-apiserver does not yet exist.") return false, nil } if err != nil { - fmt.Println("Unable to retrieve ClusterOperator/kube-apiserver:", err) - return false, err + fmt.Println("Unable to retrieve ClusterOperator/kube-apiserver (retrying):", err) + return false, nil } conditions := clusterOperator.Status.Conditions available := clusteroperatorhelpers.IsStatusConditionPresentAndEqual(conditions, configv1.OperatorAvailable, configv1.ConditionTrue) diff --git a/test/library/library.go b/test/library/library.go index 8950f5b16f..a3fe32cad9 100644 --- a/test/library/library.go +++ b/test/library/library.go @@ -15,7 +15,7 @@ import ( var ( WaitPollInterval = time.Second - WaitPollTimeout = 10 * time.Minute + WaitPollTimeout = 20 * time.Minute ) // GenerateNameForTest generates a name of the form `prefix + test name + random string` that