Skip to content

Commit 3756f25

Browse files
upodroidupodroid
committed
start running the access tokens and huge services test
1 parent 17c167a commit 3756f25

File tree

172 files changed

+805
-135
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+805
-135
lines changed

‎nodeup/pkg/model/logrotate.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func (b *LogrotateBuilder) Build(c *fi.NodeupModelBuilderContext) error {
4747
}
4848

4949
b.addLogRotate(c, "docker", "/var/log/docker.log", logRotateOptions{})
50+
b.addLogRotate(c, "kops-controller", "/var/log/kops-controller.log", logRotateOptions{})
5051
b.addLogRotate(c, "kube-addons", "/var/log/kube-addons.log", logRotateOptions{})
5152
b.addLogRotate(c, "kube-apiserver", "/var/log/kube-apiserver.log", logRotateOptions{})
5253
b.addLogRotate(c, "kube-controller-manager", "/var/log/kube-controller-manager.log", logRotateOptions{})

‎tests/e2e/kubetest2-kops/deployer/common.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"k8s.io/kops/tests/e2e/pkg/target"
3131
"k8s.io/kops/tests/e2e/pkg/util"
3232
"sigs.k8s.io/kubetest2/pkg/boskos"
33+
"sigs.k8s.io/kubetest2/pkg/exec"
3334
)
3435

3536
func (d *deployer) init() error {
@@ -112,6 +113,16 @@ func (d *deployer) initialize() error {
112113
d.GCPProject = resource.Name
113114
klog.V(1).Infof("Got project %s from boskos", d.GCPProject)
114115

116+
if os.Getenv("SCALE_SCENARIO") == "performance" {
117+
// Performance scale tests require gcloud config set project to be ran
118+
cmd := exec.Command("gcloud", "config", "set", "project", d.GCPProject)
119+
klog.V(1).Infof("Running command: %q", cmd)
120+
exec.InheritOutput(cmd)
121+
if err := cmd.Run(); err != nil {
122+
return err
123+
}
124+
}
125+
115126
if d.SSHPrivateKeyPath == "" {
116127
d.SSHPrivateKeyPath = os.Getenv("GCE_SSH_PRIVATE_KEY_FILE")
117128
}

‎tests/e2e/kubetest2-kops/deployer/up.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,7 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
168168
"--kubernetes-version", d.KubernetesVersion,
169169
"--ssh-public-key", d.SSHPublicKeyPath,
170170
"--set", "cluster.spec.nodePortAccess=0.0.0.0/0",
171-
}
172-
173-
version, err := kops.GetVersion(d.KopsBinaryPath)
174-
if err != nil {
175-
return err
176-
}
177-
if version > "1.29" {
178-
// Requires https://github.com/kubernetes/kops/pull/16128
179-
args = append(args, "--set", `spec.containerd.configAdditions=plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.runtime_type=io.containerd.runc.v2`)
171+
"--set", `spec.containerd.configAdditions=plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.runtime_type=io.containerd.runc.v2`,
180172
}
181173

182174
if yes {
@@ -264,12 +256,12 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
264256
cmd.SetEnv(d.env()...)
265257

266258
exec.InheritOutput(cmd)
267-
err = cmd.Run()
259+
err := cmd.Run()
268260
if err != nil {
269261
return err
270262
}
271263

272-
if d.setInstanceGroupOverrides(); err != nil {
264+
if err = d.setInstanceGroupOverrides(); err != nil {
273265
return err
274266
}
275267

‎tests/e2e/scenarios/scalability/run-test.sh‎

Lines changed: 63 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ if [[ -z "${K8S_VERSION:-}" ]]; then
3030
K8S_VERSION=https://storage.googleapis.com/k8s-release-dev/ci/latest.txt
3131
fi
3232

33+
# Default Scale Scenario to performance
34+
if [[ -z "${SCALE_SCENARIO:-}" ]]; then
35+
SCALE_SCENARIO="performance"
36+
export SCALE_SCENARIO
37+
fi
38+
3339
# Default cloud provider to aws
3440
if [[ -z "${CLOUD_PROVIDER:-}" ]]; then
3541
CLOUD_PROVIDER="aws"
@@ -58,17 +64,18 @@ if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
5864
create_args+=("--node-size=t3a.medium,t3.medium,t2.medium,t3a.large,c5a.large,t3.large,c5.large,m5a.large,m6a.large,m5.large,c4.large,c7a.large,r5a.large,r6a.large,m7a.large")
5965
create_args+=("--node-volume-size=20")
6066
create_args+=("--zones=us-east-2a,us-east-2b,us-east-2c")
61-
create_args+=("--image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id}")
67+
create_args+=("--image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp2/ami-id}")
6268
# TODO: track failures of tests (HostPort & OIDC) when using `--dns=none`
6369
create_args+=("--dns=none")
6470
fi
6571
if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
6672
create_args+=("--zones=us-east1-b,us-east1-c,us-east1-d")
67-
create_args+=("--node-size=e2-standard-2")
73+
create_args+=("--node-size=e2-medium")
6874
create_args+=("--node-volume-size=30")
6975
create_args+=("--master-volume-size=1000")
7076
create_args+=("--gce-service-account=default")
7177
create_args+=("--topology=private")
78+
create_args+=("--bastion")
7279
create_args+=("--image=${INSTANCE_IMAGE:-ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20251001}")
7380
create_args+=("--etcd-storage-type=hyperdisk-balanced")
7481
create_args+=("--set spec.networking.podCIDR=10.64.0.0/11")
@@ -81,11 +88,13 @@ if [[ "${CNI_PLUGIN}" == "amazonvpc" ]]; then
8188
create_args+=("--set spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true")
8289
fi
8390
create_args+=("--set spec.etcdClusters[0].manager.listenMetricsURLs=http://localhost:2382")
84-
create_args+=("--set spec.etcdClusters[0].manager.env=ETCD_QUOTA_BACKEND_BYTES=8589934592")
85-
create_args+=("--set spec.etcdClusters[1].manager.env=ETCD_QUOTA_BACKEND_BYTES=8589934592")
91+
create_args+=("--set spec.etcdClusters[*].manager.env=ETCD_QUOTA_BACKEND_BYTES=8589934592")
92+
create_args+=("--set spec.etcdClusters[*].manager.env=ETCD_ENABLE_PPROF=true")
8693
create_args+=("--set spec.cloudControllerManager.concurrentNodeSyncs=10")
8794
create_args+=("--set spec.kubelet.maxPods=96")
8895
create_args+=("--set spec.kubeScheduler.authorizationAlwaysAllowPaths=/healthz")
96+
create_args+=("--set spec.kubeScheduler.authorizationAlwaysAllowPaths=/livez")
97+
create_args+=("--set spec.kubeScheduler.authorizationAlwaysAllowPaths=/readyz")
8998
create_args+=("--set spec.kubeScheduler.authorizationAlwaysAllowPaths=/metrics")
9099
create_args+=("--set spec.kubeScheduler.kubeAPIQPS=500")
91100
create_args+=("--set spec.kubeScheduler.kubeAPIBurst=500")
@@ -118,18 +127,10 @@ create_args+=("--master-size=${CONTROL_PLANE_SIZE:-c5.2xlarge}")
118127
if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
119128
# Enable creating a single nodes instance group
120129
KOPS_FEATURE_FLAGS="AWSSingleNodesInstanceGroup,${KOPS_FEATURE_FLAGS:-}"
121-
create_args+=("--set spec.etcdClusters[0].etcdMembers[0].volumeIOPS=6000")
122-
create_args+=("--set spec.etcdClusters[0].etcdMembers[1].volumeIOPS=6000")
123-
create_args+=("--set spec.etcdClusters[0].etcdMembers[2].volumeIOPS=6000")
124-
create_args+=("--set spec.etcdClusters[0].etcdMembers[0].volumeThroughput=1000")
125-
create_args+=("--set spec.etcdClusters[0].etcdMembers[1].volumeThroughput=1000")
126-
create_args+=("--set spec.etcdClusters[0].etcdMembers[2].volumeThroughput=1000")
127-
create_args+=("--set spec.etcdClusters[0].etcdMembers[0].volumeSize=120")
128-
create_args+=("--set spec.etcdClusters[0].etcdMembers[1].volumeSize=120")
129-
create_args+=("--set spec.etcdClusters[0].etcdMembers[2].volumeSize=120")
130-
create_args+=("--set spec.etcdClusters[0].etcdMembers[0].volumeType=io1")
131-
create_args+=("--set spec.etcdClusters[0].etcdMembers[1].volumeType=io1")
132-
create_args+=("--set spec.etcdClusters[0].etcdMembers[2].volumeType=io1")
130+
create_args+=("--set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=6000")
131+
create_args+=("--set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000")
132+
create_args+=("--set spec.etcdClusters[*].etcdMembers[*].volumeSize=120")
133+
create_args+=("--set spec.etcdClusters[*].etcdMembers[*].volumeType=io1")
133134

134135
fi
135136
echo "KOPS_FEATURE_FLAGS=${KOPS_FEATURE_FLAGS}"
@@ -140,9 +141,14 @@ KUBETEST2_ARGS+=("-v=2")
140141
KUBETEST2_ARGS+=("--max-nodes-to-dump=${MAX_NODES_TO_DUMP:-5}")
141142
KUBETEST2_ARGS+=("--cloud-provider=${CLOUD_PROVIDER}")
142143
KUBETEST2_ARGS+=("--cluster-name=${CLUSTER_NAME:-}")
143-
KUBETEST2_ARGS+=("--kops-version-marker=${KOPS_VERSION_MARKER:-https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt}")
144144
KUBETEST2_ARGS+=("--admin-access=${ADMIN_ACCESS:-}")
145145
KUBETEST2_ARGS+=("--env=KOPS_FEATURE_FLAGS=${KOPS_FEATURE_FLAGS}")
146+
if [[ "${JOB_TYPE}" == "presubmit" && "${REPO_OWNER}/${REPO_NAME}" == "kubernetes/kops" ]]; then
147+
KUBETEST2_ARGS+=("--build")
148+
KUBETEST2_ARGS+=("--kops-binary-path=${GOPATH}/src/k8s.io/kops/.build/dist/linux/$(go env GOARCH)/kops")
149+
else
150+
KUBETEST2_ARGS+=("--kops-version-marker=${KOPS_VERSION_MARKER:-https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt}")
151+
fi
146152

147153
if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
148154
if [[ -n "${GCP_PROJECT:-}" ]]; then
@@ -151,10 +157,11 @@ if [[ "${CLOUD_PROVIDER}" == "gce" ]]; then
151157
KUBETEST2_ARGS+=("--boskos-resource-type=${BOSKOS_RESOURCE_TYPE:-scalability-project}")
152158
fi
153159
KUBETEST2_ARGS+=("--control-plane-instance-group-overrides=spec.rootVolume.type=hyperdisk-balanced")
160+
KUBETEST2_ARGS+=("--control-plane-instance-group-overrides=spec.associatePublicIP=true")
154161
fi
155162

156163
# More time for bigger clusters
157-
KUBETEST2_ARGS+=("--validation-wait=55m")
164+
KUBETEST2_ARGS+=("--validation-wait=75m")
158165
KUBETEST2_ARGS+=("--validation-count=3")
159166
KUBETEST2_ARGS+=("--validation-interval=60s")
160167

@@ -172,7 +179,7 @@ fi
172179
export PROMETHEUS_KUBE_PROXY_SELECTOR_KEY="k8s-app"
173180
export PROMETHEUS_SCRAPE_APISERVER_ONLY="true"
174181
export CL2_PROMETHEUS_TOLERATE_MASTER="true"
175-
if [[ "${CLOUD_PROVIDER}" == "aws" ]]; then
182+
if [[ "${CLOUD_PROVIDER}" == "aws" && "${SCALE_SCENARIO}" == "performance" ]]; then
176183
# CL2 uses KUBE_SSH_KEY_PATH path to ssh to instances for scraping metrics
177184
export KUBE_SSH_KEY_PATH="/tmp/kops/${CLUSTER_NAME}/id_ed25519"
178185
cat > "${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/load/overrides.yaml <<EOL
@@ -188,20 +195,40 @@ else
188195
EOL
189196
fi
190197

191-
kubetest2 kops "${KUBETEST2_ARGS[@]}" \
192-
--up \
193-
--kubernetes-version="${K8S_VERSION}" \
194-
--create-args="${create_args[*]}" \
195-
--test=clusterloader2 \
196-
-- \
197-
--provider="${CLOUD_PROVIDER}" \
198-
--repo-root="${GOPATH}"/src/k8s.io/perf-tests \
199-
--test-configs="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/load/config.yaml \
200-
--test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/load/overrides.yaml \
201-
--extra-args="--experimental-prometheus-snapshot-to-report-dir=true" \
202-
--kube-config="${HOME}/.kube/config"
203-
# --test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/experiments/enable_restart_count_check.yaml \
204-
# --test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/experiments/ignore_known_gce_container_restarts.yaml \
205-
# --test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/overrides/5000_nodes.yaml \
206-
# --test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/huge-service/config.yaml \
207-
# --test-overrides="${GOPATH}"/src/k8s.io/perf-tests/clusterloader2/testing/access-tokens/config.yaml \
198+
CLUSTERLOADER2_ARGS=()
199+
if [[ -n "${KOPS_CL2_TEST_CONFIG}" ]]; then
200+
CLUSTERLOADER2_ARGS+=("--test-configs=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/${KOPS_CL2_TEST_CONFIG}")
201+
else
202+
CLUSTERLOADER2_ARGS+=("--test-configs=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/load/config.yaml")
203+
CLUSTERLOADER2_ARGS+=("--test-configs=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/huge-service/config.yaml")
204+
CLUSTERLOADER2_ARGS+=("--test-configs=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/access-tokens/config.yaml")
205+
CLUSTERLOADER2_ARGS+=("--test-overrides=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/load/overrides.yaml")
206+
CLUSTERLOADER2_ARGS+=("--test-overrides=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/experiments/enable_restart_count_check.yaml")
207+
CLUSTERLOADER2_ARGS+=("--test-overrides=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/experiments/ignore_known_gce_container_restarts.yaml")
208+
CLUSTERLOADER2_ARGS+=("--test-overrides=${GOPATH}/src/k8s.io/perf-tests/clusterloader2/testing/overrides/5000_nodes.yaml")
209+
fi
210+
211+
if [[ "${SCALE_SCENARIO:performance}" == "correctness" ]]; then
212+
kubetest2 kops "${KUBETEST2_ARGS[@]}" \
213+
--up \
214+
--kubernetes-version="${K8S_VERSION}" \
215+
--create-args="${create_args[*]}" \
216+
--test=kops \
217+
-- \
218+
--test-package-url=https://storage.googleapis.com/k8s-release-dev \
219+
--test-package-dir=ci \
220+
--test-package-marker=latest.txt \
221+
--skip-regex="\[Driver:.gcepd\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:([^L].*|L[^o].*|Lo[^a].*|Loa[^d].*)\]\[KubeUp\]" \
222+
--parallel=25
223+
else
224+
kubetest2 kops "${KUBETEST2_ARGS[@]}" \
225+
--up \
226+
--kubernetes-version="${K8S_VERSION}" \
227+
--create-args="${create_args[*]}" \
228+
--test=clusterloader2 \
229+
-- \
230+
--provider="${CLOUD_PROVIDER}" \
231+
--repo-root="${GOPATH}"/src/k8s.io/perf-tests \
232+
--kube-config="${HOME}/.kube/config" \
233+
"${CLUSTERLOADER2_ARGS[@]}"
234+
fi

‎tests/integration/update_cluster/additionalobjects/data/aws_s3_object_additionalobjects.example.com-addons-bootstrap_content‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ spec:
55
addons:
66
- id: k8s-1.16
77
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
8-
manifestHash: 21c4c6429b8492f0499a751b7c319b5e9e702679df53bdf3fa285e19da46d89a
8+
manifestHash: c1db1e08d2e6e10508a09f537f8973f0911dc27fcba3230196004cba393e78ee
99
name: kops-controller.addons.k8s.io
1010
needsRollingUpdate: control-plane
1111
selector:

‎tests/integration/update_cluster/additionalobjects/data/aws_s3_object_additionalobjects.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ spec:
5656
- args:
5757
- --v=2
5858
- --conf=/etc/kubernetes/kops-controller/config/config.yaml
59+
- --log_file=/var/log/kops-controller.log
60+
- --alsologtostderr
5961
command: null
6062
env:
6163
- name: KUBERNETES_SERVICE_HOST
@@ -76,6 +78,8 @@ spec:
7678
name: kops-controller-config
7779
- mountPath: /etc/kubernetes/kops-controller/pki/
7880
name: kops-controller-pki
81+
- mountPath: /var/log/kops-controller.log
82+
name: logfile
7983
dnsPolicy: Default
8084
hostNetwork: true
8185
nodeSelector: null
@@ -98,6 +102,10 @@ spec:
98102
path: /etc/kubernetes/kops-controller/
99103
type: Directory
100104
name: kops-controller-pki
105+
- hostPath:
106+
path: /var/log/kops-controller.log
107+
type: FileOrCreate
108+
name: logfile
101109
updateStrategy:
102110
type: OnDelete
103111

‎tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-bootstrap_content‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ spec:
55
addons:
66
- id: k8s-1.16
77
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
8-
manifestHash: 7db9baec4bb7a296a0cf825e4b83b92c942f26b6a05e9e3a09d86b344e45f523
8+
manifestHash: 9c14f1d16c6a16e11f86d98b0958c3e37dc661dfee871f975a94599ff682f6be
99
name: kops-controller.addons.k8s.io
1010
needsRollingUpdate: control-plane
1111
selector:

‎tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ spec:
5656
- args:
5757
- --v=2
5858
- --conf=/etc/kubernetes/kops-controller/config/config.yaml
59+
- --log_file=/var/log/kops-controller.log
60+
- --alsologtostderr
5961
command: null
6062
env:
6163
- name: KUBERNETES_SERVICE_HOST
@@ -76,6 +78,8 @@ spec:
7678
name: kops-controller-config
7779
- mountPath: /etc/kubernetes/kops-controller/pki/
7880
name: kops-controller-pki
81+
- mountPath: /var/log/kops-controller.log
82+
name: logfile
7983
dnsPolicy: Default
8084
hostNetwork: true
8185
nodeSelector: null
@@ -98,6 +102,10 @@ spec:
98102
path: /etc/kubernetes/kops-controller/
99103
type: Directory
100104
name: kops-controller-pki
105+
- hostPath:
106+
path: /var/log/kops-controller.log
107+
type: FileOrCreate
108+
name: logfile
101109
updateStrategy:
102110
type: OnDelete
103111

‎tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_minimal.example.com-addons-bootstrap_content‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ spec:
55
addons:
66
- id: k8s-1.16
77
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
8-
manifestHash: 33c5913c89e15b5dba30a850b5ca09082d1b55f6f0af7df56f82f6f892fe310d
8+
manifestHash: 29e5ca06c3af2ea74d2c028e9c0bd5222b2884e8f2e683f21b2712042a383dab
99
name: kops-controller.addons.k8s.io
1010
needsRollingUpdate: control-plane
1111
selector:

‎tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ spec:
5656
- args:
5757
- --v=2
5858
- --conf=/etc/kubernetes/kops-controller/config/config.yaml
59+
- --log_file=/var/log/kops-controller.log
60+
- --alsologtostderr
5961
command: null
6062
env:
6163
- name: KUBERNETES_SERVICE_HOST
@@ -76,6 +78,8 @@ spec:
7678
name: kops-controller-config
7779
- mountPath: /etc/kubernetes/kops-controller/pki/
7880
name: kops-controller-pki
81+
- mountPath: /var/log/kops-controller.log
82+
name: logfile
7983
dnsPolicy: Default
8084
hostNetwork: true
8185
nodeSelector: null
@@ -98,6 +102,10 @@ spec:
98102
path: /etc/kubernetes/kops-controller/
99103
type: Directory
100104
name: kops-controller-pki
105+
- hostPath:
106+
path: /var/log/kops-controller.log
107+
type: FileOrCreate
108+
name: logfile
101109
updateStrategy:
102110
type: OnDelete
103111

0 commit comments

Comments
 (0)