From 69724d88e530c8a32566240b590409703b1d0104 Mon Sep 17 00:00:00 2001 From: Maksym Fuhol Date: Tue, 26 Nov 2024 19:51:56 +0100 Subject: [PATCH] Add example for custom compute classes reservations. (#1542) --- .../reservations/README.md | 6 ++ .../reservations/ccc-any-reservation.yaml | 30 ++++++++++ .../reservations/ccc-gpu-reservation.yaml | 35 ++++++++++++ .../reservations/ccc-local-shared-spot.yaml | 40 +++++++++++++ .../ccc-specific-reservation.yaml | 34 +++++++++++ .../reservations/ccc-ssd-reservation.yaml | 34 +++++++++++ .../reservations/deployment-gpu.yaml | 56 +++++++++++++++++++ .../reservations/deployment.yaml | 54 ++++++++++++++++++ .../reservations/reset.sh | 24 ++++++++ .../reservations/schedule.sh | 35 ++++++++++++ 10 files changed, 348 insertions(+) create mode 100644 autoscaling/custom-compute-classes/reservations/README.md create mode 100644 autoscaling/custom-compute-classes/reservations/ccc-any-reservation.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/ccc-gpu-reservation.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/ccc-local-shared-spot.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/ccc-specific-reservation.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/ccc-ssd-reservation.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/deployment-gpu.yaml create mode 100644 autoscaling/custom-compute-classes/reservations/deployment.yaml create mode 100755 autoscaling/custom-compute-classes/reservations/reset.sh create mode 100755 autoscaling/custom-compute-classes/reservations/schedule.sh diff --git a/autoscaling/custom-compute-classes/reservations/README.md b/autoscaling/custom-compute-classes/reservations/README.md new file mode 100644 index 0000000000..a3e8af58d6 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/README.md @@ -0,0 +1,6 @@ +# Custom compute classes reservations example + +[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https://github.com/GoogleCloudPlatform/kubernetes-engine-samples&cloudshell_workspace=autoscaling/custom-compute-classes/reservations) + +This example shows how to utilize custom compute classes in order to consume reservation capacity for the workloads. +Learn more about custom compute classes - https://cloud.google.com/kubernetes-engine/docs/concepts/about-custom-compute-classes diff --git a/autoscaling/custom-compute-classes/reservations/ccc-any-reservation.yaml b/autoscaling/custom-compute-classes/reservations/ccc-any-reservation.yaml new file mode 100644 index 0000000000..d2e8b76281 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/ccc-any-reservation.yaml @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_any_reservation] +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: ccc-reservations +spec: + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: ${MACHINE_TYPE} + reservations: + affinity: AnyBestEffort + whenUnsatisfiable: DoNotScaleUp +# [END gke_autoscaling_ccc_any_reservation] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/ccc-gpu-reservation.yaml b/autoscaling/custom-compute-classes/reservations/ccc-gpu-reservation.yaml new file mode 100644 index 0000000000..b552a7c4f6 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/ccc-gpu-reservation.yaml @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_gpu_reservation] +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: ccc-reservations +spec: + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: ${MACHINE_TYPE} + gpu: + type: ${ACCELERATOR_TYPE} + count: ${ACCELERATOR_COUNT} + reservations: + specific: + - name: ${RESERVATION_NAME} + affinity: Specific + whenUnsatisfiable: DoNotScaleUp +# [END gke_autoscaling_ccc_gpu_reservation] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/ccc-local-shared-spot.yaml b/autoscaling/custom-compute-classes/reservations/ccc-local-shared-spot.yaml new file mode 100644 index 0000000000..6bdee2a7bb --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/ccc-local-shared-spot.yaml @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_local_shared_spot] +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: ccc-reservations +spec: + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: ${MACHINE_TYPE} + reservations: + specific: + - name: ${RESERVATION_NAME} + affinity: Specific + - machineType: ${MACHINE_TYPE} + reservations: + specific: + - name: ${SHARED_RESERVATION_NAME + project: ${SHARED_RESERVATION_PROJECT} + affinity: Specific + - machineType: ${MACHINE_TYPE} + spot: true + whenUnsatisfiable: DoNotScaleUp +# [END gke_autoscaling_ccc_local_shared_spot] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/ccc-specific-reservation.yaml b/autoscaling/custom-compute-classes/reservations/ccc-specific-reservation.yaml new file mode 100644 index 0000000000..02de36a980 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/ccc-specific-reservation.yaml @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_specific_reservation] +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: ccc-reservations +spec: + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: ${MACHINE_TYPE} + reservations: + specific: + - name: ${RESERVATION_NAME} + - name: ${SHARED_RESERVATION_NAME + project: ${SHARED_RESERVATION_PROJECT + affinity: Specific + whenUnsatisfiable: DoNotScaleUp +# [END gke_autoscaling_ccc_specific_reservation] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/ccc-ssd-reservation.yaml b/autoscaling/custom-compute-classes/reservations/ccc-ssd-reservation.yaml new file mode 100644 index 0000000000..1fa9b2f8d2 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/ccc-ssd-reservation.yaml @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_ssd_reservation] +apiVersion: cloud.google.com/v1 +kind: ComputeClass +metadata: + name: ccc-reservations +spec: + nodePoolAutoCreation: + enabled: true + priorities: + - machineType: ${MACHINE_TYPE} + storage: + localSSDCount: ${LOCAL_SSD_COUNT} + reservations: + specific: + - name: ${RESERVATION_NAME} + affinity: Specific + whenUnsatisfiable: DoNotScaleUp +# [END gke_autoscaling_ccc_ssd_reservation] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/deployment-gpu.yaml b/autoscaling/custom-compute-classes/reservations/deployment-gpu.yaml new file mode 100644 index 0000000000..4ce799670e --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/deployment-gpu.yaml @@ -0,0 +1,56 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_reservations_deployment_gpu] +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ccc-reservations +spec: + replicas: 1 + selector: + matchLabels: + app: ccc-reservations + template: + metadata: + labels: + app: ccc-reservations + spec: + nodeSelector: + cloud.google.com/compute-class: ccc-reservations + containers: + - name: ccc-reservations + image: us-docker.pkg.dev/google-samples/containers/gke/hello-app:1.0 + resources: + requests: + cpu: 500m + memory: 500Mi + nvidia.com/gpu: ${ACCELERATOR_QUANTITY} + limits: + cpu: 500m + memory: 500Mi + nvidia.com/gpu: ${ACCELERATOR_QUANTITY} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ccc-reservations + topologyKey: "kubernetes.io/hostname" +# [END gke_autoscaling_ccc_reservations_deployment_gpu] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/deployment.yaml b/autoscaling/custom-compute-classes/reservations/deployment.yaml new file mode 100644 index 0000000000..5b6d8bed01 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/deployment.yaml @@ -0,0 +1,54 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START gke_autoscaling_ccc_reservations_deployment] +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ccc-reservations +spec: + replicas: 1 + selector: + matchLabels: + app: ccc-reservations + template: + metadata: + labels: + app: ccc-reservations + spec: + nodeSelector: + cloud.google.com/compute-class: ccc-reservations + containers: + - name: ccc-reservations + image: us-docker.pkg.dev/google-samples/containers/gke/hello-app:1.0 + resources: + requests: + cpu: 500m + memory: 500Mi + limits: + cpu: 500m + memory: 500Mi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ccc-reservations + topologyKey: "kubernetes.io/hostname" +# [END gke_autoscaling_ccc_reservations_deployment] +--- + diff --git a/autoscaling/custom-compute-classes/reservations/reset.sh b/autoscaling/custom-compute-classes/reservations/reset.sh new file mode 100755 index 0000000000..a9455f06f1 --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/reset.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +DEPLOYMENT_MANIFEST="${2:-deployment.yaml}" + +echo "Removing deployment at ${DEPLOYMENT_MANIFEST}" +kubectl delete -f "${DEPLOYMENT_MANIFEST}" --ignore-not-found=true +echo "Waiting for nodes to get downscaled..." +kubectl wait --for=delete node -l cloud.google.com/compute-class=ccc-reservations --timeout=1h diff --git a/autoscaling/custom-compute-classes/reservations/schedule.sh b/autoscaling/custom-compute-classes/reservations/schedule.sh new file mode 100755 index 0000000000..f64cdfa67d --- /dev/null +++ b/autoscaling/custom-compute-classes/reservations/schedule.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +COMPUTECLASS_MANIFEST="${1}" +DEPLOYMENT_MANIFEST="${2}" +SCALE_UP_TIMEOUT="${3:-300s}" +SCALE_UP_BATCH="${4:-1}" + +echo "Going to use deployment at ${DEPLOYMENT_MANIFEST}, compute class at ${COMPUTECLASS_MANIFEST} with batch scale up size of ${SCALE_UP_BATCH} and timeout: ${SCALE_UP_TIMEOUT}" + +kubectl apply -f "${COMPUTECLASS_MANIFEST}" +kubectl apply -f "${DEPLOYMENT_MANIFEST}" + +while :; do + kubectl rollout status -f "${DEPLOYMENT_MANIFEST}" --timeout "${SCALE_UP_TIMEOUT}" + replicas=$(kubectl get -f "${DEPLOYMENT_MANIFEST}" -o "jsonpath={.status.replicas}") + newReplicas=$(expr "${replicas}" + "${SCALE_UP_BATCH}") + echo "Scaling up deployment from ${replicas} to ${newReplicas}" + kubectl scale -f "${DEPLOYMENT_MANIFEST}" --current-replicas="${replicas}" --replicas="${newReplicas}" +done