diff --git a/config/jobs/kubernetes/sig-node/.dra-sync-settings b/config/jobs/kubernetes/sig-node/.dra-sync-settings new file mode 100644 index 000000000000..efbb28833e99 --- /dev/null +++ b/config/jobs/kubernetes/sig-node/.dra-sync-settings @@ -0,0 +1,2 @@ +# Last commit which was synced into dynamic-resource-allocation.yaml. +last_sync=056f34d0fffe6816d4f4ef7acd2e14811ff5c67a diff --git a/config/jobs/kubernetes/sig-node/dra-sync.sh b/config/jobs/kubernetes/sig-node/dra-sync.sh new file mode 100755 index 000000000000..b2e9c7e5f46b --- /dev/null +++ b/config/jobs/kubernetes/sig-node/dra-sync.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Running this script will automatically take changes made to +# dynamic-resource-allocation-canary.yaml since the last sync +# (tracked in .dra-sync-settings) and create a commit which +# applies those changes to dynamic-resource-allocation.yaml. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd -P)" +cd "${REPO_ROOT}" + +# get "last_sync" +source "config/jobs/kubernetes/sig-node/.dra-sync-settings" + +if [ -n "$(git diff --cached 2>&1)" ]; then + echo >&2 "ERROR: The git staging area must be clean." + exit 1 +fi + +new_sync=$(git rev-parse HEAD) + +diff=$(git diff ${last_sync}..${new_sync} config/jobs/kubernetes/sig-node/dynamic-resource-allocation-canary.yaml | sed -e 's/-canary//g') + +if [ -z "${diff}" ]; then + echo "No changes since last sync, nothing to do." + exit 0 +fi + +# Generate a "git format-patch" alike patch and apply it. +git am < +Date: $(date --rfc-email) +Subject: [PATCH 1/1] dra: apply changes from canary jobs + +--- +${diff} +$(diff -u config/jobs/kubernetes/sig-node/.dra-sync-settings <(sed -e "s/last_sync=.*/last_sync=${new_sync}/" config/jobs/kubernetes/sig-node/.dra-sync-settings) | sed -e 's;^--- .*;--- a/config/jobs/kubernetes/sig-node/.dra-sync-settings;' -e 's;+++ .*;+++ b/config/jobs/kubernetes/sig-node/.dra-sync-settings;') +EOF + +git log -p -n1 + +cat < + make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test" && + curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind && + kind build node-image --image=dra/node:latest . && + trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT && + kind create cluster --retain --config test/e2e/dra/kind.yaml --image dra/node:latest && + KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=1h hack/ginkgo-e2e.sh -ginkgo.label-filter='Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow' + + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + # these are both a bit below peak usage during build + # this is mostly for building kubernetes + memory: "9000Mi" + # during the tests more like 3-20m is used + cpu: 2000m + limits: + memory: "9000Mi" + cpu: 2000m + + # This jobs runs e2e.test with a focus on tests for the Dynamic Resource Allocation feature (partly alpha, partly beta) + # on a kind cluster with containerd updated to a version with CDI support. + # + # Compared to pull-kubernetes-dra, this one enables all DRA-related features. + - name: pull-kubernetes-kind-dra-all + cluster: k8s-infra-prow-build + skip_branches: + - release-\d+\.\d+ # per-release image + annotations: + testgrid-dashboards: sig-node-presubmits, sig-node-dynamic-resource-allocation + testgrid-tab-name: pr-kind-dra-all + decorate: true + path_alias: k8s.io/kubernetes + # Not relevant for most PRs. + always_run: false + # This covers most of the code related to dynamic resource allocation. + # Periodic variant: ci-kind-dra-all + run_if_changed: /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go + # The tests might still be flaky or this job might get triggered accidentally for + # an unrelated PR. + optional: true + decoration_config: + timeout: 90m + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master + command: + - runner.sh + args: + - /bin/bash + - -xc + - | + set -ex + make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test" + curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind + kind build node-image --image=dra/node:latest . + trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT + # Which DRA features exist depends on the PR that is being tested. + features=( $(grep '"DRA' pkg/features/kube_features.go | sed 's/.*"\(.*\)"/\1/') ) + echo "Enabling DRA feature(s): ${features[*]}." + # Those additional features are not in kind.yaml, but they can be added at the end. + kind create cluster --retain --config <(cat test/e2e/dra/kind.yaml; for feature in ${features}; do echo " ${feature}: true"; done) --image dra/node:latest + KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=1h hack/ginkgo-e2e.sh -ginkgo.label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Alpha, Beta, DynamicResourceAllocation$(for feature in ${features}; do echo , ${feature}; done)} && !Flaky && !Slow" + + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + # these are both a bit below peak usage during build + # this is mostly for building kubernetes + memory: "9000Mi" + # during the tests more like 3-20m is used + cpu: 2000m + limits: + memory: "9000Mi" + cpu: 2000m + + - name: pull-kubernetes-node-e2e-crio-cgrpv1-dra + cluster: k8s-infra-prow-build + skip_branches: + - release-\d+\.\d+ # per-release image + always_run: false + # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. + # CRI-O was picked because it was solid for testing so far. + # Periodic variant: ci-node-e2e-crio-cgrpv1-dra + run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) + optional: true + skip_report: false + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-pull-kubernetes-e2e: "true" + preset-pull-kubernetes-e2e-gce: "true" + annotations: + testgrid-dashboards: sig-node-cri-o, sig-node-presubmits, sig-node-dynamic-resource-allocation + testgrid-tab-name: pr-node-kubelet-crio-cgrpv1-dra + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kubernetes + extra_refs: + - org: kubernetes + repo: test-infra + base_ref: master + path_alias: k8s.io/test-infra + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master + command: + - runner.sh + - /workspace/scenarios/kubernetes_e2e.py + args: + - --deployment=node + - --env=KUBE_SSH_USER=core + - --gcp-zone=us-west1-b + - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' + - --node-tests=true + - --provider=gce + - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' + - --timeout=65m + - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv1-serial.yaml + env: + - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE + value: "1" + resources: + requests: + cpu: 4 + memory: 6Gi + limits: + cpu: 4 + memory: 6Gi + + - name: pull-kubernetes-node-e2e-crio-cgrpv2-dra + cluster: k8s-infra-prow-build + skip_branches: + - release-\d+\.\d+ # per-release image + always_run: false + # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. + # CRI-O was picked because it was solid for testing so far. + # Periodic variant: ci-node-e2e-cgrpv2-crio-dra + # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) + optional: true + skip_report: false + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + preset-pull-kubernetes-e2e: "true" + preset-pull-kubernetes-e2e-gce: "true" + annotations: + testgrid-dashboards: sig-node-cri-o, sig-node-presubmits, sig-node-dynamic-resource-allocation + testgrid-tab-name: pr-node-kubelet-crio-cgrpv2-dra + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kubernetes + extra_refs: + - org: kubernetes + repo: test-infra + base_ref: master + path_alias: k8s.io/test-infra + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master + command: + - runner.sh + - /workspace/scenarios/kubernetes_e2e.py + args: + - --deployment=node + - --env=KUBE_SSH_USER=core + - --gcp-zone=us-west1-b + - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' + - --node-tests=true + - --provider=gce + - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' + - --timeout=65m + - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2-serial.yaml + env: + - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE + value: "1" + resources: + requests: + cpu: 4 + memory: 6Gi + limits: + cpu: 4 + memory: 6Gi + + - name: pull-kubernetes-node-e2e-containerd-1-7-dra + cluster: k8s-infra-prow-build + skip_branches: + - release-\d+\.\d+ # per-release image + always_run: false + # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. + # CRI-O was picked because it was solid for testing so far. + # Periodic variant: ci-node-e2e-containerd-1-7-dra + # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) + optional: true + skip_report: false + labels: + preset-service-account: "true" + preset-k8s-ssh: "true" + annotations: + testgrid-dashboards: sig-node-presubmits, sig-node-dynamic-resource-allocation + testgrid-tab-name: pr-node-kubelet-containerd-dra + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kubernetes + extra_refs: + - org: kubernetes + repo: test-infra + base_ref: master + path_alias: k8s.io/test-infra + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master + command: + - runner.sh + - /workspace/scenarios/kubernetes_e2e.py + args: + - --deployment=node + - --gcp-zone=us-west1-b + - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/usr/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' + - --node-tests=true + - --provider=gce + - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' + - --timeout=65m + - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/dra/image-config-containerd-1.7.yaml + resources: + requests: + cpu: 4 + memory: 6Gi + limits: + cpu: 4 + memory: 6Gi diff --git a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml index c8cc801dec1d..608961381ab6 100644 --- a/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml +++ b/config/jobs/kubernetes/sig-node/sig-node-presubmit.yaml @@ -3085,121 +3085,6 @@ presubmits: cpu: 4 memory: 6Gi - # This jobs runs e2e.test with a focus on tests for the Dynamic Resource Allocation feature (currently beta) - # on a kind cluster with containerd updated to a version with CDI support. - - name: pull-kubernetes-kind-dra - cluster: k8s-infra-prow-build - skip_branches: - - release-\d+\.\d+ # per-release image - annotations: - testgrid-dashboards: sig-node-presubmits, sig-node-dynamic-resource-allocation - testgrid-tab-name: pr-kind-dra - decorate: true - path_alias: k8s.io/kubernetes - # Not relevant for most PRs. - always_run: false - # This covers most of the code related to dynamic resource allocation. - # Periodic variant: ci-kind-dra - run_if_changed: /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go - optional: true - decoration_config: - timeout: 90m - labels: - preset-service-account: "true" - preset-dind-enabled: "true" - preset-kind-volume-mounts: "true" - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - args: - - /bin/sh - - -xc - - > - make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test" && - curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind && - kind build node-image --image=dra/node:latest . && - trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT && - kind create cluster --retain --config test/e2e/dra/kind.yaml --image dra/node:latest && - KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=1h hack/ginkgo-e2e.sh -ginkgo.label-filter='Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow' - - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - # these are both a bit below peak usage during build - # this is mostly for building kubernetes - memory: "9000Mi" - # during the tests more like 3-20m is used - cpu: 2000m - limits: - memory: "9000Mi" - cpu: 2000m - - # This jobs runs e2e.test with a focus on tests for the Dynamic Resource Allocation feature (partly alpha, partly beta) - # on a kind cluster with containerd updated to a version with CDI support. - # - # Compared to pull-kubernetes-dra, this one enables all DRA-related features. - - name: pull-kubernetes-kind-dra-all - cluster: k8s-infra-prow-build - skip_branches: - - release-\d+\.\d+ # per-release image - annotations: - testgrid-dashboards: sig-node-presubmits, sig-node-dynamic-resource-allocation - testgrid-tab-name: pr-kind-dra-all - decorate: true - path_alias: k8s.io/kubernetes - # Not relevant for most PRs. - always_run: false - # This covers most of the code related to dynamic resource allocation. - # Periodic variant: ci-kind-dra-all - run_if_changed: /(dra|dynamicresources|resourceclaim|deviceclass|resourceslice|resourceclaimtemplate|dynamic-resource-allocation|pkg/apis/resource|api/resource)/.*.go - # The tests might still be flaky or this job might get triggered accidentally for - # an unrelated PR. - optional: true - decoration_config: - timeout: 90m - labels: - preset-service-account: "true" - preset-dind-enabled: "true" - preset-kind-volume-mounts: "true" - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - args: - - /bin/bash - - -xc - - | - set -ex - make WHAT="github.com/onsi/ginkgo/v2/ginkgo k8s.io/kubernetes/test/e2e/e2e.test" - curl -sSL https://kind.sigs.k8s.io/dl/latest/linux-amd64.tgz | tar xvfz - -C "${PATH%%:*}/" kind - kind build node-image --image=dra/node:latest . - trap 'kind export logs "${ARTIFACTS}/kind"; kind delete cluster' EXIT - # Which DRA features exist depends on the PR that is being tested. - features=( $(grep '"DRA' pkg/features/kube_features.go | sed 's/.*"\(.*\)"/\1/') ) - echo "Enabling DRA feature(s): ${features[*]}." - # Those additional features are not in kind.yaml, but they can be added at the end. - kind create cluster --retain --config <(cat test/e2e/dra/kind.yaml; for feature in ${features}; do echo " ${feature}: true"; done) --image dra/node:latest - KUBERNETES_PROVIDER=local KUBECONFIG=${HOME}/.kube/config GINKGO_PARALLEL_NODES=8 E2E_REPORT_DIR=${ARTIFACTS} GINKGO_TIMEOUT=1h hack/ginkgo-e2e.sh -ginkgo.label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Alpha, Beta, DynamicResourceAllocation$(for feature in ${features}; do echo , ${feature}; done)} && !Flaky && !Slow" - - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - # these are both a bit below peak usage during build - # this is mostly for building kubernetes - memory: "9000Mi" - # during the tests more like 3-20m is used - cpu: 2000m - limits: - memory: "9000Mi" - cpu: 2000m - - name: pull-kubernetes-e2e-gce-kubelet-credential-provider cluster: k8s-infra-prow-build always_run: false @@ -4169,274 +4054,6 @@ presubmits: cpu: 4 memory: 6Gi - - name: pull-kubernetes-node-e2e-crio-cgrpv1-dra - cluster: k8s-infra-prow-build - skip_branches: - - release-\d+\.\d+ # per-release image - always_run: false - # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. - # CRI-O was picked because it was solid for testing so far. - # Periodic variant: ci-node-e2e-crio-cgrpv1-dra-features - run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) - optional: true - skip_report: false - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-pull-kubernetes-e2e: "true" - preset-pull-kubernetes-e2e-gce: "true" - annotations: - testgrid-dashboards: sig-node-cri-o, sig-node-presubmits, sig-node-dynamic-resource-allocation - testgrid-tab-name: pr-node-kubelet-crio-cgrpv1-dra - decorate: true - decoration_config: - timeout: 90m - path_alias: k8s.io/kubernetes - extra_refs: - - org: kubernetes - repo: test-infra - base_ref: master - path_alias: k8s.io/test-infra - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - - /workspace/scenarios/kubernetes_e2e.py - args: - - --deployment=node - - --env=KUBE_SSH_USER=core - - --gcp-zone=us-west1-b - - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' - - --node-tests=true - - --provider=gce - - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' - - --timeout=65m - - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv1-serial.yaml - env: - - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE - value: "1" - resources: - requests: - cpu: 4 - memory: 6Gi - limits: - cpu: 4 - memory: 6Gi - - - name: pull-kubernetes-node-e2e-crio-cgrpv1-dra-kubetest2 # experimental alternative to pull-kubernetes-node-e2e-crio-cgrpv1-dra - cluster: k8s-infra-prow-build - # explicitly needs /test pull-kubernetes-node-e2e-crio-cgrpv1-dra-kubetest2 to run - always_run: false - # Don't run automatically while experimental! - # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) - optional: true - skip_report: false - skip_branches: - - release-\d+\.\d+ # per-release image - decorate: true - path_alias: k8s.io/kubernetes - extra_refs: - - org: kubernetes - repo: test-infra - base_ref: master - path_alias: k8s.io/test-infra - decoration_config: - timeout: 90m - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-pull-kubernetes-e2e: "true" - preset-pull-kubernetes-e2e-gce: "true" - annotations: - testgrid-dashboards: sig-node-cri-o, sig-node-presubmits - testgrid-tab-name: pr-node-kubelet-crio-cgrpv1-dra-kubetest2 - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - args: - - kubetest2 - - noop - - --test=node - - -- - - --repo-root=. - - --gcp-zone=us-west1-b - - --parallelism=1 - - '--test-args=--ginkgo.timeout=1h --ginkgo.label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf DynamicResourceAllocation && !Flaky && !Slow" --feature-gates="DynamicResourceAllocation=true" --service-feature-gates="DynamicResourceAllocation=true" --runtime-config=api/alpha=true,api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' - - --image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv1-serial.yaml - resources: - limits: - cpu: 4 - memory: 6Gi - requests: - cpu: 4 - memory: 6Gi - env: - - name: KUBE_SSH_USER - value: core - - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE - value: "1" - - - name: pull-kubernetes-node-e2e-crio-cgrpv2-dra - cluster: k8s-infra-prow-build - skip_branches: - - release-\d+\.\d+ # per-release image - always_run: false - # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. - # CRI-O was picked because it was solid for testing so far. - # Periodic variant: ci-node-e2e-cgrpv2-crio-dra - # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) - optional: true - skip_report: false - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-pull-kubernetes-e2e: "true" - preset-pull-kubernetes-e2e-gce: "true" - annotations: - testgrid-dashboards: sig-node-cri-o, sig-node-presubmits, sig-node-dynamic-resource-allocation - testgrid-tab-name: pr-node-kubelet-crio-cgrpv2-dra - decorate: true - decoration_config: - timeout: 90m - path_alias: k8s.io/kubernetes - extra_refs: - - org: kubernetes - repo: test-infra - base_ref: master - path_alias: k8s.io/test-infra - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - - /workspace/scenarios/kubernetes_e2e.py - args: - - --deployment=node - - --env=KUBE_SSH_USER=core - - --gcp-zone=us-west1-b - - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' - - --node-tests=true - - --provider=gce - - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' - - --timeout=65m - - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2-serial.yaml - env: - - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE - value: "1" - resources: - requests: - cpu: 4 - memory: 6Gi - limits: - cpu: 4 - memory: 6Gi - - - name: pull-kubernetes-node-e2e-crio-cgrpv2-dra-kubetest2 # experimental alternative to pull-kubernetes-node-e2e-crio-cgrpv2-dra - cluster: k8s-infra-prow-build - # explicitly needs /test pull-kubernetes-node-e2e-crio-cgrpv2-dra-kubetest2 to run - always_run: false - # Don't run automatically while experimental! - # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) - optional: true - skip_report: false - skip_branches: - - release-\d+\.\d+ # per-release image - decorate: true - path_alias: k8s.io/kubernetes - extra_refs: - - org: kubernetes - repo: test-infra - base_ref: master - path_alias: k8s.io/test-infra - decoration_config: - timeout: 90m - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - preset-pull-kubernetes-e2e: "true" - preset-pull-kubernetes-e2e-gce: "true" - annotations: - testgrid-dashboards: sig-node-cri-o, sig-node-presubmits - testgrid-tab-name: pr-node-kubelet-crio-cgrpv2-dra-kubetest2 - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - args: - - kubetest2 - - noop - - --test=node - - -- - - --repo-root=. - - --gcp-zone=us-west1-b - - --parallelism=1 - - '--test-args=--ginkgo.timeout=1h --ginkgo.label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf DynamicResourceAllocation && !Flaky && !Slow" --feature-gates="DynamicResourceAllocation=true" --service-feature-gates="DynamicResourceAllocation=true" --runtime-config=api/alpha=true,api/beta=true --container-runtime-endpoint=unix:///var/run/crio/crio.sock --container-runtime-process-name=/usr/local/bin/crio --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/crio.service --kubelet-cgroups=/system.slice/kubelet.service" --extra-log="{\"name\": \"crio.log\", \"journalctl\": [\"-u\", \"crio\"]}"' - - --image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/crio/latest/image-config-cgroupv2-serial.yaml - resources: - limits: - cpu: 4 - memory: 6Gi - requests: - cpu: 4 - memory: 6Gi - env: - - name: KUBE_SSH_USER - value: core - - name: IGNITION_INJECT_GCE_SSH_PUBLIC_KEY_FILE - value: "1" - - name: pull-kubernetes-node-e2e-containerd-1-7-dra - cluster: k8s-infra-prow-build - skip_branches: - - release-\d+\.\d+ # per-release image - always_run: false - # Automatically testing with one container runtime in one configuration is sufficient to detect basic problems in kubelet early. - # CRI-O was picked because it was solid for testing so far. - # Periodic variant: ci-node-e2e-containerd-1-7-dra - # run_if_changed: (/dra/|/dynamicresources/|/resourceclaim/|/deviceclass/|/resourceslice/|/resourceclaimtemplate/|/dynamic-resource-allocation/|/pkg/apis/resource/|/api/resource/|/test/e2e_node/dra_).*\.(go|yaml) - optional: true - skip_report: false - labels: - preset-service-account: "true" - preset-k8s-ssh: "true" - annotations: - testgrid-dashboards: sig-node-presubmits, sig-node-dynamic-resource-allocation - testgrid-tab-name: pr-node-kubelet-containerd-dra - decorate: true - decoration_config: - timeout: 90m - path_alias: k8s.io/kubernetes - extra_refs: - - org: kubernetes - repo: test-infra - base_ref: master - path_alias: k8s.io/test-infra - spec: - containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241128-8df65c072f-master - command: - - runner.sh - - /workspace/scenarios/kubernetes_e2e.py - args: - - --deployment=node - - --gcp-zone=us-west1-b - - '--node-test-args=--feature-gates=DynamicResourceAllocation=true --service-feature-gates=DynamicResourceAllocation=true --runtime-config=api/beta=true --container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/usr/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd\"]}"' - - --node-tests=true - - --provider=gce - - '--test_args=--timeout=1h --label-filter="Feature: containsAny DynamicResourceAllocation && Feature: isSubsetOf { Beta, DynamicResourceAllocation } && !Flaky && !Slow"' - - --timeout=65m - - --node-args=--image-config-file=/home/prow/go/src/k8s.io/test-infra/jobs/e2e_node/dra/image-config-containerd-1.7.yaml - resources: - requests: - cpu: 4 - memory: 6Gi - limits: - cpu: 4 - memory: 6Gi - - name: pull-kubernetes-node-e2e-resource-health-status cluster: k8s-infra-prow-build skip_branches: diff --git a/config/jobs/kubernetes/sig-node/sig_node_test.go b/config/jobs/kubernetes/sig-node/sig_node_test.go new file mode 100644 index 000000000000..a0259e188fc6 --- /dev/null +++ b/config/jobs/kubernetes/sig-node/sig_node_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package policy + +// This file validates Kubernetes's jobs configs against policies. + +import ( + "encoding/json" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "sigs.k8s.io/prow/pkg/config" + "sigs.k8s.io/yaml" +) + +func TestYaml(t *testing.T) { + if err := filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { + if !strings.HasSuffix(path, ".yaml") { + return nil + } + t.Run(path, func(t *testing.T) { + var content struct { + Presets []config.Preset `json:"presets,omitempty"` + Templates any `json:"templates,omitempty"` + Periodics []config.Periodic `json:"periodics,omitempty"` + PreSubmits map[string][]config.Presubmit `json:"presubmits,omitempty"` + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + // Because of https://github.com/kubernetes-sigs/yaml/issues/46, + // strict parsing cannot be used as it would complain about + // repeated keys in maps, which is intentional and (in YAML) + // valid when using aliases. + if err := yaml.Unmarshal(data, &content, func(d *json.Decoder) *json.Decoder { + d.DisallowUnknownFields() + return d + }); err != nil { + t.Fatalf("error unmarshaling data: %v", err) + } + + // The Templates are just helpers, what matters are the jobs. + content.Templates = nil + + data, err = yaml.Marshal(&content) + if err != nil { + t.Fatalf("error re-marshaling content: %v", err) + } + t.Logf("\n%s", string(data)) + }) + return nil + }); err != nil { + t.Errorf("Error looking for YAML files: %v", err) + } +}