diff --git a/.github/workflows/gke-vpa-recommendations-ci.yml b/.github/workflows/gke-vpa-recommendations-ci.yml new file mode 100644 index 0000000000..2f60ffbc51 --- /dev/null +++ b/.github/workflows/gke-vpa-recommendations-ci.yml @@ -0,0 +1,25 @@ +name: gke-vpa-recommendations-ci +on: + push: + branches: + - main + paths: + - '.github/workflows/gke-vpa-recommendations-ci.yml' + - 'gke-vpa-recommendations/**' + pull_request: + paths: + - '.github/workflows/gke-vpa-recommendations-ci.yml' + - 'gke-vpa-recommendations/**' +jobs: + job: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: build metric-exporter container + run: | + cd gke-vpa-recommendations/metrics-exporter + docker build --tag metric-exporter . + - name: build hpa-metrics-exporter container + run: | + cd gke-vpa-recommendations/scripts + docker build --tag hpa-metrics-exporter . diff --git a/.gitignore b/.gitignore index c2edbca0f2..bc19e2ccb4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ # Misc. .DS_Store -.idea/ \ No newline at end of file +.idea/ diff --git a/CODEOWNERS b/CODEOWNERS index dd018c8288..92a0038a5b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,4 +8,5 @@ # Directory-specific owners /windows-multi-arch/ @ibabou @liurupeng @mcshooter @pjh /gke-stateful-postgres/ @aburhan @pwschuurman +/gke-vpa-recommendations/ @aburhan /whereami/ @theemadnes diff --git a/gke-vpa-recommendations/README.md b/gke-vpa-recommendations/README.md new file mode 100644 index 0000000000..5edd356f22 --- /dev/null +++ b/gke-vpa-recommendations/README.md @@ -0,0 +1,4 @@ +# Right-size your GKE workloads at scale + +See step by step tutorial at [Right-size your GKE workloads at scale] +(https://cloud.google.com/kubernetes-engine/docs/tutorials/right-size-workloads-at-scale) \ No newline at end of file diff --git a/gke-vpa-recommendations/k8s/online-shop.yaml b/gke-vpa-recommendations/k8s/online-shop.yaml new file mode 100644 index 0000000000..c23b5207b7 --- /dev/null +++ b/gke-vpa-recommendations/k8s/online-shop.yaml @@ -0,0 +1,886 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ---------------------------------------------------------- +# WARNING: This file is autogenerated. Do not manually edit. +# ---------------------------------------------------------- + +# [START gke_release_kubernetes_manifests_microservices_demo] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emailservice +spec: + selector: + matchLabels: + app: emailservice + template: + metadata: + labels: + app: emailservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/emailservice:v0.4.1 + ports: + - containerPort: 8080 + env: + - name: PORT + value: "8080" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + readinessProbe: + periodSeconds: 5 + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + livenessProbe: + periodSeconds: 5 + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + resources: + requests: + cpu: 15m + memory: 138Mi + limits: + cpu: 30m + memory: 500Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: emailservice +spec: + type: ClusterIP + selector: + app: emailservice + ports: + - name: grpc + port: 5000 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: checkoutservice +spec: + selector: + matchLabels: + app: checkoutservice + template: + metadata: + labels: + app: checkoutservice + spec: + serviceAccountName: default + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/checkoutservice:v0.4.1 + ports: + - containerPort: 5050 + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:5050"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:5050"] + env: + - name: PORT + value: "5050" + - name: PRODUCT_CATALOG_SERVICE_ADDR + value: "productcatalogservice:3550" + - name: SHIPPING_SERVICE_ADDR + value: "shippingservice:50051" + - name: PAYMENT_SERVICE_ADDR + value: "paymentservice:50051" + - name: EMAIL_SERVICE_ADDR + value: "emailservice:5000" + - name: CURRENCY_SERVICE_ADDR + value: "currencyservice:7000" + - name: CART_SERVICE_ADDR + value: "cartservice:7070" + - name: DISABLE_STATS + value: "1" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + resources: + requests: + cpu: 10m + memory: 20Mi + limits: + cpu: 25m + memory: 40Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: checkoutservice +spec: + type: ClusterIP + selector: + app: checkoutservice + ports: + - name: grpc + port: 5050 + targetPort: 5050 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: recommendationservice +spec: + selector: + matchLabels: + app: recommendationservice + template: + metadata: + labels: + app: recommendationservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/recommendationservice:v0.4.1 + ports: + - containerPort: 8080 + readinessProbe: + periodSeconds: 5 + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + livenessProbe: + periodSeconds: 5 + exec: + command: ["/bin/grpc_health_probe", "-addr=:8080"] + env: + - name: PORT + value: "8080" + - name: PRODUCT_CATALOG_SERVICE_ADDR + value: "productcatalogservice:3550" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + - name: DISABLE_DEBUGGER + value: "1" + resources: + requests: + cpu: 10m + memory: 22Mi + limits: + cpu: 20m + memory: 45Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: recommendationservice +spec: + type: ClusterIP + selector: + app: recommendationservice + ports: + - name: grpc + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend +spec: + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + annotations: + sidecar.istio.io/rewriteAppHTTPProbers: "true" + spec: + serviceAccountName: default + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/frontend:v0.4.1 + ports: + - containerPort: 8080 + readinessProbe: + initialDelaySeconds: 10 + httpGet: + path: "/_healthz" + port: 8080 + httpHeaders: + - name: "Cookie" + value: "shop_session-id=x-readiness-probe" + livenessProbe: + initialDelaySeconds: 10 + httpGet: + path: "/_healthz" + port: 8080 + httpHeaders: + - name: "Cookie" + value: "shop_session-id=x-liveness-probe" + env: + - name: PORT + value: "8080" + - name: PRODUCT_CATALOG_SERVICE_ADDR + value: "productcatalogservice:3550" + - name: CURRENCY_SERVICE_ADDR + value: "currencyservice:7000" + - name: CART_SERVICE_ADDR + value: "cartservice:7070" + - name: RECOMMENDATION_SERVICE_ADDR + value: "recommendationservice:8080" + - name: SHIPPING_SERVICE_ADDR + value: "shippingservice:50051" + - name: CHECKOUT_SERVICE_ADDR + value: "checkoutservice:5050" + - name: AD_SERVICE_ADDR + value: "adservice:9555" + # # ENV_PLATFORM: One of: local, gcp, aws, azure, onprem, alibaba + # # When not set, defaults to "local" unless running in GKE, otherwies auto-sets to gcp + # - name: ENV_PLATFORM + # value: "aws" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + # - name: CYMBAL_BRANDING + # value: "true" + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend +spec: + type: ClusterIP + selector: + app: frontend + ports: + - name: http + port: 80 + targetPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-external +spec: + type: LoadBalancer + selector: + app: frontend + ports: + - name: http + port: 80 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: paymentservice +spec: + selector: + matchLabels: + app: paymentservice + template: + metadata: + labels: + app: paymentservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/paymentservice:v0.4.1 + ports: + - containerPort: 50051 + env: + - name: PORT + value: "50051" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + - name: DISABLE_DEBUGGER + value: "1" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 100m + memory: 64Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: paymentservice +spec: + type: ClusterIP + selector: + app: paymentservice + ports: + - name: grpc + port: 50051 + targetPort: 50051 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productcatalogservice +spec: + selector: + matchLabels: + app: productcatalogservice + template: + metadata: + labels: + app: productcatalogservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/productcatalogservice:v0.4.1 + ports: + - containerPort: 3550 + env: + - name: PORT + value: "3550" + - name: DISABLE_STATS + value: "1" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:3550"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:3550"] + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 100m + memory: 64Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: productcatalogservice +spec: + type: ClusterIP + selector: + app: productcatalogservice + ports: + - name: grpc + port: 3550 + targetPort: 3550 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cartservice +spec: + selector: + matchLabels: + app: cartservice + template: + metadata: + labels: + app: cartservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/cartservice:v0.4.1 + ports: + - containerPort: 7070 + env: + - name: REDIS_ADDR + value: "redis-cart:6379" + resources: + requests: + cpu: 200m + memory: 64Mi + limits: + cpu: 300m + memory: 128Mi + readinessProbe: + initialDelaySeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"] + livenessProbe: + initialDelaySeconds: 15 + periodSeconds: 10 + exec: + command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"] +--- +apiVersion: v1 +kind: Service +metadata: + name: cartservice +spec: + type: ClusterIP + selector: + app: cartservice + ports: + - name: grpc + port: 7070 + targetPort: 7070 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loadgenerator +spec: + selector: + matchLabels: + app: loadgenerator + replicas: 1 + template: + metadata: + labels: + app: loadgenerator + annotations: + sidecar.istio.io/rewriteAppHTTPProbers: "true" + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + restartPolicy: Always + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + initContainers: + - command: + - /bin/sh + - -exc + - | + echo "Init container pinging frontend: ${FRONTEND_ADDR}..." + STATUSCODE=$(wget --server-response http://${FRONTEND_ADDR} 2>&1 | awk '/^ HTTP/{print $2}') + if test $STATUSCODE -ne 200; then + echo "Error: Could not reach frontend - Status code: ${STATUSCODE}" + exit 1 + fi + name: frontend-check + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: busybox:latest + env: + - name: FRONTEND_ADDR + value: "frontend:80" + containers: + - name: main + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/loadgenerator:v0.4.1 + env: + - name: FRONTEND_ADDR + value: "frontend:80" + - name: USERS + value: "10" + resources: + requests: + cpu: 300m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: currencyservice +spec: + selector: + matchLabels: + app: currencyservice + template: + metadata: + labels: + app: currencyservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/currencyservice:v0.4.1 + ports: + - name: grpc + containerPort: 7000 + env: + - name: PORT + value: "7000" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + - name: DISABLE_DEBUGGER + value: "1" + readinessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:7000"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:7000"] + +--- +apiVersion: v1 +kind: Service +metadata: + name: currencyservice +spec: + type: ClusterIP + selector: + app: currencyservice + ports: + - name: grpc + port: 7000 + targetPort: 7000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shippingservice +spec: + selector: + matchLabels: + app: shippingservice + template: + metadata: + labels: + app: shippingservice + spec: + serviceAccountName: default + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/shippingservice:v0.4.1 + ports: + - containerPort: 50051 + env: + - name: PORT + value: "50051" + - name: DISABLE_STATS + value: "1" + - name: DISABLE_TRACING + value: "1" + - name: DISABLE_PROFILER + value: "1" + readinessProbe: + periodSeconds: 5 + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + livenessProbe: + exec: + command: ["/bin/grpc_health_probe", "-addr=:50051"] + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: shippingservice +spec: + type: ClusterIP + selector: + app: shippingservice + ports: + - name: grpc + port: 50051 + targetPort: 50051 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-cart +spec: + selector: + matchLabels: + app: redis-cart + template: + metadata: + labels: + app: redis-cart + spec: + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: redis + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: redis:alpine + ports: + - containerPort: 6379 + readinessProbe: + periodSeconds: 5 + tcpSocket: + port: 6379 + livenessProbe: + periodSeconds: 5 + tcpSocket: + port: 6379 + volumeMounts: + - mountPath: /data + name: redis-data + resources: + limits: + memory: 50Mi + cpu: 50m + requests: + cpu: 50m + memory: 50Mi + volumes: + - name: redis-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: redis-cart +spec: + type: ClusterIP + selector: + app: redis-cart + ports: + - name: tcp-redis + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: adservice +spec: + selector: + matchLabels: + app: adservice + template: + metadata: + labels: + app: adservice + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: server + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + privileged: false + readOnlyRootFilesystem: true + image: gcr.io/google-samples/microservices-demo/adservice:v0.4.1 + ports: + - containerPort: 9555 + env: + - name: PORT + value: "9555" + - name: DISABLE_STATS + value: "1" + - name: DISABLE_TRACING + value: "1" + resources: + requests: + cpu: 200m + memory: 88Mi + limits: + cpu: 300m + memory: 154Mi + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:9555"] + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:9555"] +--- +apiVersion: v1 +kind: Service +metadata: + name: adservice +spec: + type: ClusterIP + selector: + app: adservice + ports: + - name: grpc + port: 9555 + targetPort: 9555 +# [END gke_release_kubernetes_manifests_microservices_demo] \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/.dockerignore b/gke-vpa-recommendations/metrics-exporter/.dockerignore new file mode 100644 index 0000000000..60dc03071e --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/.dockerignore @@ -0,0 +1,8 @@ +Dockerfile +/terraform +README.md +*.pyc +*.pyo +*.pyd +__pycache__ +.pytest_cache \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/Dockerfile b/gke-vpa-recommendations/metrics-exporter/Dockerfile new file mode 100644 index 0000000000..40f9db523a --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/Dockerfile @@ -0,0 +1,7 @@ + +FROM python:3.9 +WORKDIR /app +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt +COPY . . +CMD ["python", "./main.py"] \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/Procfile b/gke-vpa-recommendations/metrics-exporter/Procfile new file mode 100644 index 0000000000..eb399cbcf2 --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/Procfile @@ -0,0 +1 @@ +web: python3 main.py \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/cloudbuild.yaml b/gke-vpa-recommendations/metrics-exporter/cloudbuild.yaml new file mode 100644 index 0000000000..aa672d845a --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/cloudbuild.yaml @@ -0,0 +1,31 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file and other cloudbuild.yaml files are used to ensure that +# our public Docker images such as gcr.io/google-samples/hello-app:1.0 +# are rebuilt and updated upon changes to the repository. + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '-t' + - 'us-docker.pkg.dev/google-samples/containers/gke/metrics-exporter' + - '.' + dir: 'gke-vpa-recommendations/metrics-exporter' + +# Push images. +images: + - 'us-docker.pkg.dev/google-samples/containers/gke/metrics-exporter' + diff --git a/gke-vpa-recommendations/metrics-exporter/config.py b/gke-vpa-recommendations/metrics-exporter/config.py new file mode 100644 index 0000000000..b0d7b57082 --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/config.py @@ -0,0 +1,65 @@ +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +PROJECT_ID = os.getenv("PROJECT_ID", "") +PUBSUB_TOPIC = "mql_metric_export" +BIGQUERY_DATASET = "metric_export" +BIGQUERY_TABLE = "mql_metrics" +RECOMMENDATION_TABLE = "vpa_container_recommendations" +RECOMMENDATION_WINDOW_SECONDS = 2592000 +LATEST_WINDOW_SECONDS = 60 + +# IMPORTANT: to guarantee successfully retriving data, please use a time window greater than 5 minutes + +MQL_QUERY = { + "container_count" :["kubernetes.io/container/cpu/request_cores", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "cpu_requested_cores" :["kubernetes.io/container/cpu/request_cores", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "cpu_limit_cores": ["kubernetes.io/container/cpu/limit_cores", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "memory_requested_bytes":["kubernetes.io/container/memory/request_bytes", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "memory_limit_bytes":["kubernetes.io/container/memory/limit_bytes", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "memory_request_recommendations": ["kubernetes.io/autoscaler/container/memory/per_replica_recommended_request_bytes", RECOMMENDATION_WINDOW_SECONDS, "vpa_metric"] +, + "cpu_request_recommendations": ["kubernetes.io/autoscaler/container/cpu/per_replica_recommended_request_cores", RECOMMENDATION_WINDOW_SECONDS, "vpa_metric"] +, + "hpa_cpu":["custom.googleapis.com/podautoscaler/hpa/cpu/target_utilization", LATEST_WINDOW_SECONDS, "gke_metric"] +, + "hpa_memory":["custom.googleapis.com/podautoscaler/hpa/memory/target_utilization", LATEST_WINDOW_SECONDS, "gke_metric"] + +} + +BASE_URL = "https://monitoring.googleapis.com/v3/projects" +QUERY_URL = f"{BASE_URL}/{PROJECT_ID}/timeSeries:query" + + +BQ_VALUE_MAP = { + "INT64": "int64_value", + "BOOL": "boolean_value", + "DOUBLE": "double_value", + "STRING": "string_value", + "DISTRIBUTION": "distribution_value" +} + +API_VALUE_MAP = { + "INT64": "int64Value", + "BOOL": "booleanValue", + "DOUBLE": "doubleValue", + "STRING": "stringValue", + "DISTRIBUTION": "distributionValue" +} diff --git a/gke-vpa-recommendations/metrics-exporter/main.py b/gke-vpa-recommendations/metrics-exporter/main.py new file mode 100644 index 0000000000..21a2753b4b --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/main.py @@ -0,0 +1,300 @@ +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time +import config +import logging +import numpy as np +from google.cloud import bigquery +from google.cloud import bigquery_storage_v1 +from google.cloud.bigquery_storage_v1 import types +from google.cloud.bigquery_storage_v1 import writer +from google.protobuf import descriptor_pb2 +import metric_record_flat_pb2 +from google.cloud import monitoring_v3 +import math +import os + +# If you update the metric_record.proto protocol buffer definition, run: +# +# protoc --python_out=. metric_record_flat.proto +# +# from the samples/snippets directory to generate the metric_record_pb2.py module. + +# Fetch GKE metrics - cpu requested cores, cpu limit cores, memory requested bytes, memory limit bytes, count and all workloads with hpa +def get_gke_metrics(metric_name, metric, window): + output = [] + client = monitoring_v3.MetricServiceClient() + project_name = f"projects/{config.PROJECT_ID}" + now = time.time() + seconds = int(now) + nanos = int((now - seconds) * 10 ** 9) + gke_group_by_fields = [ 'resource.label."location"','resource.label."project_id"','resource.label."cluster_name"','resource.label."controller_name"','resource.label."namespace_name"','metadata.system_labels."top_level_controller_name"','metadata.system_labels."top_level_controller_type"'] + hpa_group_by_fields = ['resource.label."location"','resource.label."project_id"','resource.label."cluster_name"','resource.label."namespace_name"','metric.label."targetref_kind"','metric.label."targetref_name"'] + + interval = monitoring_v3.TimeInterval( + { + "end_time": {"seconds": seconds, "nanos": nanos}, + "start_time": {"seconds": (seconds - window), "nanos": nanos}, + } + ) + + aggregation = monitoring_v3.Aggregation( + { + "alignment_period": {"seconds": window}, + "per_series_aligner": monitoring_v3.Aggregation.Aligner.ALIGN_MEAN, + "cross_series_reducer": monitoring_v3.Aggregation.Reducer.REDUCE_COUNT if metric_name == "container_count" else monitoring_v3.Aggregation.Reducer.REDUCE_MEAN, + "group_by_fields": gke_group_by_fields if "hpa" not in metric_name else hpa_group_by_fields, + } + ) + try: + results = client.list_time_series( + request={ + "name": project_name, + "filter": f'metric.type = "{metric}" AND resource.label.namespace_name != "kube-system"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, + "aggregation": aggregation, + } + ) + print("Building Row") + + for result in results: + row = metric_record_flat_pb2.MetricFlatRecord () + label = result.resource.labels + metadata = result.metadata.system_labels.fields + metricdata = result.metric.labels + row.metric_name = metric_name + row.location = label['location'] + row.project_id = label['project_id'] + row.cluster_name = label['cluster_name'] + row.controller_name = metricdata['targetref_name'] if "hpa" in metric_name else metadata['top_level_controller_name'].string_value + row.controller_type= metricdata['targetref_kind'] if "hpa" in metric_name else metadata['top_level_controller_type'].string_value + row.namespace_name = label['namespace_name'] + row.tstamp = time.time() + points = result.points + print(row) + for point in points: + if "cpu" in metric_name: + row.points = (int(point.value.double_value * 1000)) if point.value.double_value is not None else 0 + elif "memory" in metric_name: + row.points = (int(point.value.double_value/1024/1024)) if point.value.double_value is not None else 0 + else: + row.points = (point.value.int64_value) if point.value.int64_value is not None else 0 + break + output.append(row.SerializeToString()) + + except: + print("No HPA workloads found") + + return output + +# Build VPA recommendations, memory: get max value over 30 days, cpu: get max and 95th percentile +def get_vpa_recommenation_metrics(metric_name, metric, window): + + from google.cloud import monitoring_v3 + client = monitoring_v3.MetricServiceClient() + project_name = f"projects/{config.PROJECT_ID}" + interval = monitoring_v3.TimeInterval() + + now = time.time() + seconds = int(now) + nanos = int((now - seconds) * 10 ** 9) + + interval = monitoring_v3.TimeInterval( + { + "end_time": {"seconds": seconds, "nanos": nanos}, + "start_time": {"seconds": (seconds - window), "nanos": nanos}, + } + ) + + results = client.list_time_series( + request={ + "name": project_name, + "filter": f'metric.type = "{metric}" AND resource.label.namespace_name != "kube-system"', + "interval": interval, + "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, + } + ) + output = [] + + for result in results: + points_array = [] + row = metric_record_flat_pb2.MetricFlatRecord () + label = result.resource.labels + row.location = label['location'] + row.project_id = label['project_id'] + row.cluster_name = label['cluster_name'] + row.controller_name = label['controller_name'] + row.controller_type= label['controller_kind'] + row.namespace_name = label['namespace_name'] + row.tstamp = time.time() + for point in result.points: + if((point.value.double_value) != 0): + points_array.append(int(point.value.double_value * 1000)) + else: + points_array.append(int(point.value.int64_value/1024/1024)) + if "cpu" in metric_name : + row.metric_name = "cpu_request_95th_percentile_recommendations" + row.points = int(np.percentile(points_array, 95)) + output.append(row.SerializeToString()) + row.metric_name = "cpu_request_max_recommendations" + row.points = (max(points_array)) + output.append(row.SerializeToString()) + else: + row.metric_name = metric_name + row.points = (max(points_array)) + output.append(row.SerializeToString()) + + return output + + +# Write rows to BigQuery +def append_rows_proto(rows): + + """Create a write stream, write some sample data, and commit the stream.""" + write_client = bigquery_storage_v1.BigQueryWriteClient() + parent = write_client.table_path(config.PROJECT_ID, config.BIGQUERY_DATASET, config.BIGQUERY_TABLE) + write_stream = types.WriteStream() + + # When creating the stream, choose the type. Use the PENDING type to wait + # until the stream is committed before it is visible. See: + # https://cloud.google.com/bigquery/docs/reference/storage/rpc/google.cloud.bigquery.storage.v1#google.cloud.bigquery.storage.v1.WriteStream.Type + write_stream.type_ = types.WriteStream.Type.PENDING + write_stream = write_client.create_write_stream( + parent=parent, write_stream=write_stream + ) + stream_name = write_stream.name + + # Create a template with fields needed for the first request. + request_template = types.AppendRowsRequest() + + # The initial request must contain the stream name. + request_template.write_stream = stream_name + + # So that BigQuery knows how to parse the serialized_rows, generate a + # protocol buffer representation of your message descriptor. + proto_schema = types.ProtoSchema() + proto_descriptor = descriptor_pb2.DescriptorProto() + metric_record_flat_pb2.MetricFlatRecord.DESCRIPTOR.CopyToProto(proto_descriptor) + proto_schema.proto_descriptor = proto_descriptor + proto_data = types.AppendRowsRequest.ProtoData() + proto_data.writer_schema = proto_schema + request_template.proto_rows = proto_data + + # Some stream types support an unbounded number of requests. Construct an + # AppendRowsStream to send an arbitrary number of requests to a stream. + append_rows_stream = writer.AppendRowsStream(write_client, request_template) + + # Create a batch of row data by appending proto2 serialized bytes to the + # serialized_rows repeated field. + proto_rows = types.ProtoRows() + for row in rows: + proto_rows.serialized_rows.append(row) + request = types.AppendRowsRequest() + request.offset = 0 + proto_data = types.AppendRowsRequest.ProtoData() + proto_data.rows = proto_rows + request.proto_rows = proto_data + + append_rows_stream.send(request) + + # Shutdown background threads and close the streaming connection. + append_rows_stream.close() + + # A PENDING type stream must be "finalized" before being committed. No new + # records can be written to the stream after this method has been called. + write_client.finalize_write_stream(name=write_stream.name) + + # Commit the stream you created earlier. + batch_commit_write_streams_request = types.BatchCommitWriteStreamsRequest() + batch_commit_write_streams_request.parent = parent + batch_commit_write_streams_request.write_streams = [write_stream.name] + write_client.batch_commit_write_streams(batch_commit_write_streams_request) + + print(f"Writes to stream: '{write_stream.name}' have been committed.") + +# Purge all data from metrics table. mql_metrics table is used as a staging table and must be purged to avoid duplicate metrics +def purge_raw_metric_data(): + t = time.time() - 5400 + client = bigquery.Client() + metric_table_id = f'{config.PROJECT_ID}.{config.BIGQUERY_DATASET}.{config.BIGQUERY_TABLE}' + + purge_raw_metric_query_job=client.query( + f"""DELETE `{metric_table_id}` WHERE TRUE AND tstamp < {t} + """ + ) + print("Raw metric data purged from {}".format(metric_table_id)) + purge_raw_metric_query_job.result() + +# Use recommendation.sql to build vpa container recommendations +def build_recommenation_table(): + """ Create recommenations table in BigQuery + """ + client = bigquery.Client() + + + table_id = f'{config.PROJECT_ID}.{config.BIGQUERY_DATASET}.{config.RECOMMENDATION_TABLE}' + + update_query = f"""UPDATE `{table_id}` + SET latest = FALSE + WHERE latest = TRUE""" + query_job = client.query(update_query) + query_job.result() + + with open('./recommendation.sql', 'r') as file: + sql = file.read() + sql = sql.replace("PROJECT_ID",config.PROJECT_ID) + + print("Query results loaded to the table {}".format(table_id)) + + # Start the query, passing in the recommendation query. + query_job = client.query(sql) # Make an API request. + query_job.result() # Wait for the job to complete. + purge_raw_metric_data() + +def run_pipeline(): + + for metric, query in config.MQL_QUERY.items(): + if query[2] == "gke_metric": + print(f"Processing GKE system metric {metric}") + append_rows_proto(get_gke_metrics(metric, query[0], query[1])) + else: + print(f"Processing VPA recommendation metric {metric}") + append_rows_proto(get_vpa_recommenation_metrics(metric, query[0], query[1])) + build_recommenation_table() + + +def export_metric_data(event, context): + """Background Cloud Function to be triggered by Pub/Sub. + Args: + event (dict): The dictionary with data specific to this type of + event. The `data` field contains the PubsubMessage message. The + `attributes` field will contain custom attributes if there are any. + context (google.cloud.functions.Context): The Cloud Functions event + metadata. The `event_id` field contains the Pub/Sub message ID. The + `timestamp` field contains the publish time. + """ + print("""This Function was triggered by messageId {} published at {} + """.format(context.event_id, context.timestamp)) + if 'PROJECT_ID' not in os.environ or not os.environ['PROJECT_ID']: + print("Please set the 'PROJECT_ID' environment variable.") + else: + run_pipeline() + + +if __name__ == "__main__": + if 'PROJECT_ID' not in os.environ or not os.environ['PROJECT_ID']: + print("Please set the 'PROJECT_ID' environment variable.") + else: + run_pipeline() \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/metric_record_flat.proto b/gke-vpa-recommendations/metrics-exporter/metric_record_flat.proto new file mode 100644 index 0000000000..6b18a26ea4 --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/metric_record_flat.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +message MetricFlatRecord { + + string metric_name = 1; + string location = 2; + string project_id = 3; + string cluster_name = 4; + string controller_name = 5; + string controller_type = 6; + string namespace_name = 7; + int64 points = 8; + float tstamp = 9; +} \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/metric_record_flat_pb2.py b/gke-vpa-recommendations/metrics-exporter/metric_record_flat_pb2.py new file mode 100644 index 0000000000..144f142769 --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/metric_record_flat_pb2.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: metric_record_flat.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='metric_record_flat.proto', + package='', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x18metric_record_flat.proto\"\xcd\x01\n\x10MetricFlatRecord\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x10\n\x08location\x18\x02 \x01(\t\x12\x12\n\nproject_id\x18\x03 \x01(\t\x12\x14\n\x0c\x63luster_name\x18\x04 \x01(\t\x12\x17\n\x0f\x63ontroller_name\x18\x05 \x01(\t\x12\x17\n\x0f\x63ontroller_type\x18\x06 \x01(\t\x12\x16\n\x0enamespace_name\x18\x07 \x01(\t\x12\x0e\n\x06points\x18\x08 \x01(\x03\x12\x0e\n\x06tstamp\x18\t \x01(\x02\x62\x06proto3' +) + + + + +_METRICFLATRECORD = _descriptor.Descriptor( + name='MetricFlatRecord', + full_name='MetricFlatRecord', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='metric_name', full_name='MetricFlatRecord.metric_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='location', full_name='MetricFlatRecord.location', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='project_id', full_name='MetricFlatRecord.project_id', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cluster_name', full_name='MetricFlatRecord.cluster_name', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='controller_name', full_name='MetricFlatRecord.controller_name', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='controller_type', full_name='MetricFlatRecord.controller_type', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='namespace_name', full_name='MetricFlatRecord.namespace_name', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='points', full_name='MetricFlatRecord.points', index=7, + number=8, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='tstamp', full_name='MetricFlatRecord.tstamp', index=8, + number=9, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=29, + serialized_end=234, +) + +DESCRIPTOR.message_types_by_name['MetricFlatRecord'] = _METRICFLATRECORD +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +MetricFlatRecord = _reflection.GeneratedProtocolMessageType('MetricFlatRecord', (_message.Message,), { + 'DESCRIPTOR' : _METRICFLATRECORD, + '__module__' : 'metric_record_flat_pb2' + # @@protoc_insertion_point(class_scope:MetricFlatRecord) + }) +_sym_db.RegisterMessage(MetricFlatRecord) + + +# @@protoc_insertion_point(module_scope) diff --git a/gke-vpa-recommendations/metrics-exporter/recommendation.sql b/gke-vpa-recommendations/metrics-exporter/recommendation.sql new file mode 100644 index 0000000000..b4f7e7f398 --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/recommendation.sql @@ -0,0 +1,217 @@ +/* +# Copyright 2022 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/ +INSERT +INTO + `PROJECT_ID.metric_export.vpa_container_recommendations` + ( +recommendation_timestamp, +location, +project_id, +cluster_name, +controller_name, +controller_type, +namespace_name, +container_count, +cpu_limit_cores, +cpu_requested_cores, +memory_limit_bytes, +memory_requested_bytes, +memory_request_max_recommendations, +mem_qos, +cpu_qos, +memory_limit_recommendations, +cpu_request_recommendations, +cpu_limit_recommendations, +cpu_delta, +mem_delta, +priority, +mem_provision_status, +mem_provision_risk, +cpu_provision_status, +cpu_provision_risk, +latest +) +############################### +# Gather all HPA workloads +############################## +WITH + hpa_workloads AS ( + SELECT + location, + project_id, + cluster_name, + controller_name, + controller_type, + namespace_name, + 1 AS flag + FROM + `PROJECT_ID.metric_export.mql_metrics` + WHERE + metric_name LIKE '%hpa%' ), +################################################### +# Filter out HPA workloads, convert rows to columns +################################################### + workloads_without_hpa AS ( + SELECT + *, + TIMESTAMP(TIMESTAMP_SECONDS(CAST(tstamp AS INT64))) AS recommendation_timestamp, + FROM ( + SELECT + DISTINCT(metric_name), + c.location, + c.project_id, + c.cluster_name, + c.controller_name, + c.controller_type, + c.namespace_name, + IF((c.points IS NULL), 0, c.points) AS points, + LAST_VALUE(c.tstamp) OVER (PARTITION BY c.controller_name, c.project_id, c.cluster_name, c.location ORDER BY c.tstamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS tstamp, + FROM + `PROJECT_ID.metric_export.mql_metrics` AS c + LEFT JOIN + hpa_workloads + ON + c.controller_name = hpa_workloads.controller_name + AND c.project_id = hpa_workloads.project_id + AND c.location = hpa_workloads.location + AND c.cluster_name = hpa_workloads.cluster_name + WHERE + hpa_workloads.flag IS NULL + ORDER BY + metric_name) PIVOT(AVG(points) FOR metric_name IN ( 'container_count', + 'memory_requested_bytes', + 'memory_limit_bytes', + 'memory_request_recommendations', + 'cpu_requested_cores', + 'cpu_limit_cores', + 'cpu_request_95th_percentile_recommendations', + 'cpu_request_max_recommendations'))), +############################### +# QoS +############################## +qos AS ( + SELECT + * EXCEPT (tstamp), + CASE + WHEN (memory_requested_bytes = 0 AND memory_limit_bytes = 0) THEN 'BestEffort' + WHEN (memory_requested_bytes = memory_limit_bytes) + AND (memory_requested_bytes> 0) THEN 'Guaranteed' + ELSE + 'Burstable' + END + AS mem_qos, + CASE + WHEN (cpu_requested_cores = 0 AND cpu_limit_cores = 0) THEN 'BestEffort' + WHEN (cpu_requested_cores = cpu_limit_cores) + AND (cpu_requested_cores > 0) THEN 'Guaranteed' + ELSE + 'Burstable' + END + AS cpu_qos, + FROM + workloads_without_hpa + WHERE + memory_request_recommendations IS NOT NULL + AND (cpu_request_max_recommendations IS NOT NULL + OR cpu_request_95th_percentile_recommendations IS NOT NULL ) ), +############################################################## +# Use QoS to determine the CPU recommendations +############################################################## +recommendation AS ( +SELECT * EXCEPT (cpu_request_95th_percentile_recommendations, cpu_request_max_recommendations), +memory_request_recommendations AS memory_limit_recommendation, +IF(cpu_qos = "Guaranteed", cpu_request_max_recommendations, cpu_request_95th_percentile_recommendations ) as cpu_request_recommendations, +CASE + WHEN (cpu_limit_cores = 0 or cpu_requested_cores = 0 ) THEN cpu_request_max_recommendations + WHEN (cpu_qos = "Guaranteed" ) THEN cpu_request_max_recommendations + ELSE + CAST(cpu_request_95th_percentile_recommendations * (cpu_limit_cores/cpu_requested_cores) AS INT64) + END + AS cpu_limit_recommendation +FROM qos +), +############################################################## +# Build final recommendation query with prority and advisory +############################################################## +final_recommendation AS ( + SELECT * , +( IF(cpu_requested_cores IS NULL, 0, cpu_requested_cores) - cpu_request_recommendations ) AS cpu_delta, +( memory_requested_bytes - memory_request_recommendations ) AS mem_delta, +CAST(container_count * ((cpu_requested_cores - cpu_request_recommendations) + (memory_requested_bytes - memory_request_recommendations)/13.4) AS INT64) AS priority, + CASE + WHEN (memory_requested_bytes > memory_request_recommendations) THEN "over" + WHEN (memory_requested_bytes < memory_request_recommendations) THEN "under" + WHEN (memory_requested_bytes = 0) THEN "not set" + ELSE + "ok" +END + AS mem_provision_status, + CASE + WHEN (memory_requested_bytes > memory_request_recommendations) THEN "cost" + WHEN (memory_requested_bytes < memory_request_recommendations) THEN "reliability" + WHEN (memory_requested_bytes = 0) THEN "reliability" + ELSE + "ok" +END + AS mem_provision_risk, + CASE + WHEN (cpu_requested_cores > cpu_request_recommendations) THEN "over" + WHEN (cpu_requested_cores < cpu_request_recommendations) THEN "under" + WHEN (cpu_requested_cores = 0 ) THEN "not set" + ELSE + "ok" +END + AS cpu_provision_status, + CASE + WHEN (cpu_requested_cores > cpu_request_recommendations) THEN "cost" + WHEN (cpu_requested_cores < cpu_request_recommendations) THEN "performance" + WHEN (cpu_requested_cores = 0) THEN "reliability" + ELSE + "ok" +END + AS cpu_provision_risk, +TRUE as latest +FROM recommendation +) + +SELECT +recommendation_timestamp, +location, +project_id, +cluster_name, +controller_name, +controller_type, +namespace_name, +CAST(container_count AS INT64), +CAST(cpu_limit_cores AS INT64), +CAST(cpu_requested_cores AS INT64), +CAST(memory_limit_bytes AS INT64), +CAST(memory_requested_bytes AS INT64), +CAST(memory_request_recommendations AS INT64) as memory_request_max_recommendations, +mem_qos, +cpu_qos, +CAST(memory_limit_recommendation AS INT64), +CAST(cpu_request_recommendations AS INT64), +CAST(cpu_limit_recommendation AS INT64), +CAST(cpu_delta AS INT64), +CAST(mem_delta AS INT64), +CAST(priority AS INT64), +mem_provision_status, +mem_provision_risk, +cpu_provision_status, +cpu_provision_risk, +latest FROM final_recommendation WHERE priority IS NOT NULL +ORDER BY priority DESC \ No newline at end of file diff --git a/gke-vpa-recommendations/metrics-exporter/requirements.txt b/gke-vpa-recommendations/metrics-exporter/requirements.txt new file mode 100644 index 0000000000..6c3e50c78f --- /dev/null +++ b/gke-vpa-recommendations/metrics-exporter/requirements.txt @@ -0,0 +1,24 @@ +functions-framework==3.* +cachetools +certifi +cffi +chardet +google-api-core +google-auth +google-cloud-bigquery +google-cloud-bigquery-storage +google-cloud-monitoring +google-cloud-core +google-crc32c +google-resumable-media +idna +protobuf +pyasn1 +pyasn1-modules +pycparser +pytz +requests +rsa +six +urllib3 +numpy \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/Dockerfile b/gke-vpa-recommendations/scripts/Dockerfile new file mode 100644 index 0000000000..cfb3caec80 --- /dev/null +++ b/gke-vpa-recommendations/scripts/Dockerfile @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.15-alpine as builder +WORKDIR /go/src/vpa-metrics-extporter +COPY . . +RUN go build -o /export . + +FROM alpine as release +RUN apk add --update --no-cache curl && \ + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + mv kubectl /usr/bin/kubectl && \ + chmod +x /usr/bin/kubectl +COPY --from=builder /export /export +CMD [ "/export" ] \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/README.md b/gke-vpa-recommendations/scripts/README.md new file mode 100644 index 0000000000..ace29c2230 --- /dev/null +++ b/gke-vpa-recommendations/scripts/README.md @@ -0,0 +1,3 @@ +# Monitoring your GKE clusters for cost-optimization using Cloud Monitoring + +See step by step tutorial at [Monitoring your GKE clusters for cost-optimization using Cloud Monitoring](https://cloud.google.com/architecture/monitoring-gke-clusters-for-cost-optimization-using-cloud-monitoring) diff --git a/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa.go b/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa.go new file mode 100644 index 0000000000..01c8ab233b --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa.go @@ -0,0 +1,233 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "fmt" + + v1 "k8s.io/api/autoscaling/v1" + "k8s.io/api/autoscaling/v2beta1" + "k8s.io/api/autoscaling/v2beta2" +) + +//DecodeHPAList just for backward compatibility. Use version specific ones +func DecodeHPAList(data []byte) ([]HPA, error) { + return DecodeHPAListV1(data) +} + +//DecodeHPAListV1 reads k8s HorizontalPodAutoScaler yaml and trasform to HPA v1 object +func DecodeHPAListV1(data []byte) ([]HPA, error) { + scheme := buildHPAListV1Scheme() + obj, _, err := decode(scheme, data) + if err != nil { + return []HPA{}, fmt.Errorf("Error Decoding. Root cause %+v", err) + } + return buildHPAListV1(obj), nil +} + +//DecodeHPAListV2Beta1 reads k8s HorizontalPodAutoScaler yaml and trasform to HPA v2beta1 object +func DecodeHPAListV2Beta1(data []byte) ([]HPA, error) { + scheme := buildHPAListV2Beta1Scheme() + obj, _, err := decode(scheme, data) + if err != nil { + return []HPA{}, fmt.Errorf("Error Decoding. Root cause %+v", err) + } + return buildHPAListV2Beta1(obj), nil +} + +//DecodeHPAListV2Beta2 reads k8s HorizontalPodAutoScaler yaml and trasform to HPA v2beta2 object +func DecodeHPAListV2Beta2(data []byte) ([]HPA, error) { + scheme := buildHPAListV2Beta2Scheme() + obj, _, err := decode(scheme, data) + if err != nil { + return []HPA{}, fmt.Errorf("Error Decoding. Root cause %+v", err) + } + return buildHPAListV2Beta2(obj), nil +} + +func buildHPAListV1(obj interface{}) []HPA { + list := []HPA{} + vpaList := obj.(*v1.HorizontalPodAutoscalerList) + for _, v := range vpaList.Items { + gvk := v.GetObjectKind().GroupVersionKind() + hpa, err := buildHPA(&v, GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}) + if err != nil { + fmt.Printf("Unable to decode object %+v. Root cause: %+v", gvk, err) + } else { + list = append(list, hpa) + } + } + return list +} + +func buildHPAListV2Beta1(obj interface{}) []HPA { + list := []HPA{} + vpaList := obj.(*v2beta1.HorizontalPodAutoscalerList) + for _, v := range vpaList.Items { + gvk := v.GetObjectKind().GroupVersionKind() + hpa, err := buildHPA(&v, GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}) + if err != nil { + fmt.Printf("Unable to decode object %+v. Root cause: %+v", gvk, err) + } else { + list = append(list, hpa) + } + } + return list +} + +func buildHPAListV2Beta2(obj interface{}) []HPA { + list := []HPA{} + vpaList := obj.(*v2beta2.HorizontalPodAutoscalerList) + for _, v := range vpaList.Items { + gvk := v.GetObjectKind().GroupVersionKind() + hpa, err := buildHPA(&v, GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}) + if err != nil { + fmt.Printf("Unable to decode object %+v. Root cause: %+v", gvk, err) + } else { + list = append(list, hpa) + } + } + return list +} + +//decodeHPA reads k8s HorizontalPodAutoScaler yaml and trasform to HPA object - mostly used by tests +func decodeHPA(data []byte) (HPA, error) { + scheme := buildHPAScheme() + obj, gvk, err := decode(scheme, data) + if err != nil { + return HPA{}, fmt.Errorf("Error Decoding. Check if your GroupVersionKind is defined in api/k8s_decoder.go. Root cause %+v", err) + } + return buildHPA(obj, gvk) +} + +//buildHPA reads k8s HorizontalPodAutoScaler object and trasform to HPA object +func buildHPA(obj interface{}, groupVersionKind GroupVersionKind) (HPA, error) { + switch obj.(type) { + case *v2beta2.HorizontalPodAutoscaler: + return buildHPAV2beta2(obj.(*v2beta2.HorizontalPodAutoscaler)), nil + case *v2beta1.HorizontalPodAutoscaler: + return buildHPAV2beta1(obj.(*v2beta1.HorizontalPodAutoscaler)), nil + case *v1.HorizontalPodAutoscaler: + return buildHPAV1(obj.(*v1.HorizontalPodAutoscaler)), nil + default: + return HPA{}, fmt.Errorf("APIVersion and Kind not Implemented: %+v", groupVersionKind) + } +} + +func buildHPAV2beta2(hpa *v2beta2.HorizontalPodAutoscaler) HPA { + var targetCPUPercentage int32 = 0 + var targetMemoryPercentage int32 = 0 + netrics := hpa.Spec.Metrics + for i := 0; i < len(netrics); i++ { + metric := netrics[i] + if metric.Type == v2beta2.ResourceMetricSourceType { + res := metric.Resource + target := res.Target + if res.Name == "cpu" && target.AverageUtilization != (*int32)(nil) { + targetCPUPercentage = *target.AverageUtilization + } else if res.Name == "memory" && target.AverageUtilization != (*int32)(nil) { + targetMemoryPercentage = *target.AverageUtilization + } + } + } + + var minReplicas int32 = 1 + if hpa.Spec.MinReplicas != (*int32)(nil) { + minReplicas = *hpa.Spec.MinReplicas + } + + namespace := "default" + if hpa.GetNamespace() != "" { + namespace = hpa.GetNamespace() + } + + tr := hpa.Spec.ScaleTargetRef + return HPA{ + Namespace: namespace, + Name: hpa.GetName(), + TargetRef: TargetRef{APIVersion: tr.APIVersion, Kind: tr.Kind, Name: tr.Name}, + MinReplicas: minReplicas, + MaxReplicas: hpa.Spec.MaxReplicas, + TargetCPUPercentage: targetCPUPercentage, + TargetMemoryPercentage: targetMemoryPercentage, + } +} + +func buildHPAV2beta1(hpa *v2beta1.HorizontalPodAutoscaler) HPA { + var targetCPUPercentage int32 = 0 + var targetMemoryPercentage int32 = 0 + netrics := hpa.Spec.Metrics + for i := 0; i < len(netrics); i++ { + metric := netrics[i] + if metric.Type == v2beta1.ResourceMetricSourceType { + res := metric.Resource + if res.Name == "cpu" && res.TargetAverageUtilization != (*int32)(nil) { + targetCPUPercentage = *res.TargetAverageUtilization + } else if res.Name == "memory" && res.TargetAverageUtilization != (*int32)(nil) { + targetMemoryPercentage = *res.TargetAverageUtilization + } + } + } + + var minReplicas int32 = 1 + if hpa.Spec.MinReplicas != (*int32)(nil) { + minReplicas = *hpa.Spec.MinReplicas + } + + namespace := "default" + if hpa.GetNamespace() != "" { + namespace = hpa.GetNamespace() + } + + tr := hpa.Spec.ScaleTargetRef + return HPA{ + Namespace: namespace, + Name: hpa.GetName(), + TargetRef: TargetRef{APIVersion: tr.APIVersion, Kind: tr.Kind, Name: tr.Name}, + MinReplicas: minReplicas, + MaxReplicas: hpa.Spec.MaxReplicas, + TargetCPUPercentage: targetCPUPercentage, + TargetMemoryPercentage: targetMemoryPercentage, + } +} + +func buildHPAV1(hpa *v1.HorizontalPodAutoscaler) HPA { + var targetCPUPercentage int32 = 0 + var targetMemoryPercentage int32 = 0 + if hpa.Spec.TargetCPUUtilizationPercentage != (*int32)(nil) { + targetCPUPercentage = *hpa.Spec.TargetCPUUtilizationPercentage + } + + var minReplicas int32 = 1 + if hpa.Spec.MinReplicas != (*int32)(nil) { + minReplicas = *hpa.Spec.MinReplicas + } + + namespace := "default" + if hpa.GetNamespace() != "" { + namespace = hpa.GetNamespace() + } + + tr := hpa.Spec.ScaleTargetRef + return HPA{ + Namespace: namespace, + Name: hpa.GetName(), + TargetRef: TargetRef{APIVersion: tr.APIVersion, Kind: tr.Kind, Name: tr.Name}, + MinReplicas: minReplicas, + MaxReplicas: hpa.Spec.MaxReplicas, + TargetCPUPercentage: targetCPUPercentage, + TargetMemoryPercentage: targetMemoryPercentage, + } +} diff --git a/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa_test.go b/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa_test.go new file mode 100644 index 0000000000..a754120fd7 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/builder_hpa_test.go @@ -0,0 +1,604 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "fmt" + "io/ioutil" + "strings" + "testing" +) + +func TestHPAAPINotImplemented(t *testing.T) { + yaml := `apiVersion: autoscaling/V10000 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + minReplicas: 10 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60` + + _, err := decodeHPA([]byte(yaml)) + if err == nil || !strings.HasPrefix(err.Error(), "Error Decoding.") { + t.Error(fmt.Errorf("Should have return an APIVersion error, but returned '%+v'", err)) + } +} + +func TestHPABasicV2beta2(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + minReplicas: 10 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expected := "default" + if got := hpa.Namespace; got != expected { + t.Errorf("Expected Namespace %+v, got %+v", expected, got) + } + expected = "php-apache" + if got := hpa.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + expected = "apps/v1" + if got := hpa.TargetRef.APIVersion; got != expected { + t.Errorf("Expected APIVersion %+v, got %+v", expected, got) + } + expected = "Deployment" + if got := hpa.TargetRef.Kind; got != expected { + t.Errorf("Expected Kind %+v, got %+v", expected, got) + } + expected = "php-apache" + if got := hpa.TargetRef.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + + expectedMinReplicas := int32(10) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } + + expectedMaxReplicas := int32(20) + if got := hpa.MaxReplicas; got != expectedMaxReplicas { + t.Errorf("Expected Max Replicas %+v, got %+v", expectedMaxReplicas, got) + } + + expectedCPU := int32(60) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } + expectedMemory := int32(0) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPANoMinReplicasV2beta2(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedMinReplicas := int32(1) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } +} + +func TestHPATargetMemoryV2beta2(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: test-memory-hpa +spec: + scaleTargetRef: + apiVersion: "apps/v1" + kind: Deployment + name: test + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if hpa.TargetCPUPercentage != 0 { + t.Error("Target CPU should be zero") + } + expectedMemory := int32(60) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPATargetCPUAndMemoryV2beta2(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: test-memory-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: test + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedCPU := int32(50) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } + expectedMemory := int32(60) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPANoTargetCPUV2beta2(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + minReplicas: 10 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache` + + hpa, _ := decodeHPA([]byte(yaml)) + if hpa.TargetCPUPercentage != 0 { + t.Error("Target CPU should be zero") + } +} + +func TestHPABasicV2beta1(t *testing.T) { + yaml := ` +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: frontend-scaler +spec: + scaleTargetRef: + kind: Deployment + name: frobinator-frontend + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 80` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expected := "default" + if got := hpa.Namespace; got != expected { + t.Errorf("Expected Namespace %+v, got %+v", expected, got) + } + expected = "frontend-scaler" + if got := hpa.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + expected = "" + if got := hpa.TargetRef.APIVersion; got != expected { + t.Errorf("Expected APIVersion %+v, got %+v", expected, got) + } + expected = "Deployment" + if got := hpa.TargetRef.Kind; got != expected { + t.Errorf("Expected Kind %+v, got %+v", expected, got) + } + expected = "frobinator-frontend" + if got := hpa.TargetRef.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + + expectedMinReplicas := int32(2) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } + + expectedMaxReplicas := int32(10) + if got := hpa.MaxReplicas; got != expectedMaxReplicas { + t.Errorf("Expected Max Replicas %+v, got %+v", expectedMaxReplicas, got) + } + + expectedCPU := int32(80) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } + expectedMemory := int32(0) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPANoMinReplicasV2beta1(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 80` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedMinReplicas := int32(1) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } +} + +func TestHPATargetMemoryV2beta1(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: test-memory-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: test + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: memory + targetAverageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if hpa.TargetCPUPercentage != 0 { + t.Error("Target CPU should be zero") + } + expectedMemory := int32(60) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPATargetCPUAndMemoryV2beta1(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: test-memory-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: test + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageUtilization: 60` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedCPU := int32(50) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } + expectedMemory := int32(60) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPANoTargetCPUVV2Beta1(t *testing.T) { + yaml := `apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + maxReplicas: 20 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + maxReplicas: 10` + + hpa, _ := decodeHPA([]byte(yaml)) + if hpa.TargetCPUPercentage != 0 { + t.Error("Target CPU should be zero") + } +} + +func TestHPABasicV1(t *testing.T) { + yaml := ` +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app: adservice + name: adservice +spec: + minReplicas: 5 + maxReplicas: 20 + scaleTargetRef: + kind: Deployment + name: adservice + targetCPUUtilizationPercentage: 80` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expected := "default" + if got := hpa.Namespace; got != expected { + t.Errorf("Expected Namespace %+v, got %+v", expected, got) + } + expected = "adservice" + if got := hpa.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + expected = "" + if got := hpa.TargetRef.APIVersion; got != expected { + t.Errorf("Expected APIVersion %+v, got %+v", expected, got) + } + expected = "Deployment" + if got := hpa.TargetRef.Kind; got != expected { + t.Errorf("Expected Kind %+v, got %+v", expected, got) + } + expected = "adservice" + if got := hpa.TargetRef.Name; got != expected { + t.Errorf("Expected Name %+v, got %+v", expected, got) + } + + expectedMinReplicas := int32(5) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } + + expectedMaxReplicas := int32(20) + if got := hpa.MaxReplicas; got != expectedMaxReplicas { + t.Errorf("Expected Max Replicas %+v, got %+v", expectedMaxReplicas, got) + } + + expectedCPU := int32(80) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } + expectedMemory := int32(0) + if got := hpa.TargetMemoryPercentage; got != expectedMemory { + t.Errorf("Expected target Memory %+v, got %+v", expectedMemory, got) + } +} + +func TestHPANoMinReplicasV1(t *testing.T) { + yaml := ` +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app: adservice + name: adservice +spec: + maxReplicas: 20 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: adservice + targetCPUUtilizationPercentage: 80` + + hpa, err := decodeHPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedMinReplicas := int32(1) + if got := hpa.MinReplicas; got != expectedMinReplicas { + t.Errorf("Expected Min Replicas %+v, got %+v", expectedMinReplicas, got) + } + + expectedMaxReplicas := int32(20) + if got := hpa.MaxReplicas; got != expectedMaxReplicas { + t.Errorf("Expected Max Replicas %+v, got %+v", expectedMaxReplicas, got) + } + + expectedCPU := int32(80) + if got := hpa.TargetCPUPercentage; got != expectedCPU { + t.Errorf("Expected target CPU %+v, got %+v", expectedCPU, got) + } +} + +func TestHPANoTargetCPUVV1(t *testing.T) { + yaml := ` +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + labels: + app: adservice + name: adservice +spec: + maxReplicas: 20 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: adservice` + + hpa, _ := decodeHPA([]byte(yaml)) + if hpa.TargetCPUPercentage != 0 { + t.Error("Target CPU should be zero") + } +} + +func TestHPADecodeListV2Beta2(t *testing.T) { + data, err := ioutil.ReadFile("./testdata/hpa-v2beta2.yaml") + if err != nil { + t.Errorf("Error reading file: %+v", err) + } + + hpas, err := DecodeHPAListV2Beta2(data) + if err != nil { + t.Errorf("Error decoding file: %+v", err) + } + for _, hpa := range hpas { + if hpa.TargetCPUPercentage == 0 && hpa.TargetMemoryPercentage == 0 { + t.Errorf("HPA should have resource cpu or memory utilization: %+v", hpa) + } + } +} + +func TestHPADecodeListV2Beta1(t *testing.T) { + data, err := ioutil.ReadFile("./testdata/hpa-v2beta1.yaml") + if err != nil { + t.Errorf("Error reading file: %+v", err) + } + + hpas, err := DecodeHPAListV2Beta1(data) + if err != nil { + t.Errorf("Error decoding file: %+v", err) + } + for _, hpa := range hpas { + if hpa.TargetCPUPercentage == 0 && hpa.TargetMemoryPercentage == 0 { + t.Errorf("HPA should have resource cpu or memory utilization: %+v", hpa) + } + } +} + +func TestHPADecodeListV1(t *testing.T) { + data, err := ioutil.ReadFile("./testdata/hpa-v1.yaml") + if err != nil { + t.Errorf("Error reading file: %+v", err) + } + + hpas, err := DecodeHPAListV1(data) + if err != nil { + t.Errorf("Error decoding file: %+v", err) + } + noUtilizationDefined := 0 + for _, hpa := range hpas { + if hpa.TargetCPUPercentage == 0 && hpa.TargetMemoryPercentage == 0 { + noUtilizationDefined++ + } + } + if noUtilizationDefined != 1 { + t.Errorf("HPA should have resource cpu or memory utilization for all but one") + } +} diff --git a/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa.go b/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa.go new file mode 100644 index 0000000000..6be6856d3f --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa.go @@ -0,0 +1,112 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "fmt" + + v1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +//DecodeVPAList reads k8s VerticalPodAutoScaler yaml and trasform to VPA object +func DecodeVPAList(data []byte) ([]VPA, error) { + scheme := buildVPAScheme() + obj, gvk, err := decode(scheme, data) + if err != nil { + return []VPA{}, fmt.Errorf("Error Decoding. Check if your GroupVersionKind is defined in api/k8s/decoder.go. Root cause %+v", err) + } + switch obj.(type) { + case *v1.VerticalPodAutoscalerList: + return buildVPAList(obj), nil + default: + return []VPA{}, fmt.Errorf("APIVersion and Kind not Supported: %+v", gvk) + } +} + +func buildVPAList(obj interface{}) []VPA { + list := []VPA{} + vpaList := obj.(*v1.VerticalPodAutoscalerList) + for _, v := range vpaList.Items { + gvk := v.GetObjectKind().GroupVersionKind() + vpa, err := buildVPA(&v, GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}) + if err != nil { + fmt.Printf("Unable to decode object %+v. Root cause: %+v", gvk, err) + } else { + list = append(list, vpa) + } + } + return list +} + +//decodeVPA reads k8s VerticalPodAutoScaler yaml and trasform to VPA object +func decodeVPA(data []byte) (VPA, error) { + scheme := buildVPAScheme() + obj, gvk, err := decode(scheme, data) + if err != nil { + return VPA{}, fmt.Errorf("Error Decoding. Check if your GroupVersionKind is defined in api/k8s_decoder.go. Root cause %+v", err) + } + return buildVPA(obj, gvk) +} + +//buildVPA reads k8s VerticalPodAutoScaler object and trasform to VPA object +func buildVPA(obj interface{}, gvk GroupVersionKind) (VPA, error) { + switch gvk.Version { + case gvkVPAV1.Version: + return buildVPAV1(obj.(*v1.VerticalPodAutoscaler)), nil + case gvkVPAV1beta2.Version: + return buildVPAV1(obj.(*v1.VerticalPodAutoscaler)), nil + default: + return VPA{}, fmt.Errorf("APIVersion and Kind not Supported: %+v", gvk) + } +} + +func buildVPAV1(vpa *v1.VerticalPodAutoscaler) VPA { + recomendations := []VPARecomendation{} + if vpa.Status.Recommendation != nil { + cr := vpa.Status.Recommendation.ContainerRecommendations + for _, r := range cr { + recomendation := VPARecomendation{ + ContainerName: r.ContainerName, + Target: Resource{ + CPU: float32(r.Target.Cpu().MilliValue()) / 1000, + Memory: r.Target.Memory().Value(), + }, + UpperBound: Resource{ + CPU: float32(r.UpperBound.Cpu().MilliValue()) / 1000, + Memory: r.UpperBound.Memory().Value(), + }, + } + recomendations = append(recomendations, recomendation) + } + } + + isInRecomendationMode := false // default is Auto + if vpa.Spec.UpdatePolicy != nil { + isInRecomendationMode = *vpa.Spec.UpdatePolicy.UpdateMode == v1.UpdateModeOff + } + + tr := vpa.Spec.TargetRef + ns := "default" + if vpa.GetNamespace() != "" { + ns = vpa.GetNamespace() + } + return VPA{ + Namespace: ns, + Name: vpa.GetName(), + TargetRef: TargetRef{APIVersion: tr.APIVersion, Kind: tr.Kind, Name: tr.Name}, + IsInRecomendationMode: isInRecomendationMode, + Recomendations: recomendations, + } +} diff --git a/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa_test.go b/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa_test.go new file mode 100644 index 0000000000..55d19c0f51 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/builder_vpa_test.go @@ -0,0 +1,569 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "fmt" + "io/ioutil" + "strings" + "testing" +) + +func TestVPAV1beta1NotImplemented(t *testing.T) { + yaml := ` +apiVersion: autoscaling.k8s.io/v1beta1 +kind: VerticalPodAutoscaler +metadata: + name: redis-vpa +spec: + selector: + matchLabels: + label: vpa-label + updatePolicy: + updateMode: "Off"` + + _, err := decodeVPA([]byte(yaml)) + if err == nil || !strings.HasPrefix(err.Error(), "APIVersion and Kind not Supported") { + t.Error(fmt.Errorf("Should have return an APIVersion error, but returned '%+v'", err)) + } +} + +func TestVPAV1WithoutStatus(t *testing.T) { + yaml := ` +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: redis-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: redis-master` + + vpa, err := decodeVPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if vpa.IsInRecomendationMode { + t.Error("VPA object is in Auto mode") + } + + if len(vpa.Recomendations) > 0 { + t.Error("VPA object has no recomendation.") + } +} + +func TestVPAV1WithStatus(t *testing.T) { + yaml := ` +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: redis-vpa + namespace: otherns +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: redis-master + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 35m + memory: 262144k + uncappedTarget: + cpu: 35m + memory: 262144k + upperBound: + cpu: 39m + memory: 262144k` + + vpa, err := decodeVPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedNS := "otherns" + if got := vpa.Namespace; got != expectedNS { + t.Errorf("Expected Namespace %+v, got %+v", expectedNS, got) + } + + expectedName := "redis-vpa" + if got := vpa.Name; got != expectedName { + t.Errorf("Expected Name %+v, got %+v", expectedName, got) + } + + expectedTargetRef := TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "redis-master"} + if got := vpa.TargetRef; got != expectedTargetRef { + t.Errorf("Expected TargetRef %+v, got %+v", expectedTargetRef, got) + } + + if !vpa.IsInRecomendationMode { + t.Error("VPA object is in recomendation mode") + } + + recomendation := vpa.Recomendations[0] + expectedCPU := float32(0.035) + expectedMemory := int64(262144000) + if recomendation.Target.CPU != expectedCPU { + t.Errorf("Expected Target Recomended CPU %+v, got %+v", expectedCPU, recomendation.Target.CPU) + } + if recomendation.Target.Memory != expectedMemory { + t.Errorf("Expected Target Recomended Memory %+v, got %+v", expectedMemory, recomendation.Target.Memory) + } + expectedCPU = float32(0.039) + expectedMemory = int64(262144000) + if recomendation.UpperBound.CPU != expectedCPU { + t.Errorf("Expected UpperBound Recomended CPU %+v, got %+v", expectedCPU, recomendation.UpperBound.CPU) + } + if recomendation.UpperBound.Memory != expectedMemory { + t.Errorf("Expected UpperBound Recomended Memory %+v, got %+v", expectedMemory, recomendation.UpperBound.Memory) + } +} + +func TestVPAV1WithManyContainers(t *testing.T) { + yaml := ` +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: redis-vpa + namespace: otherns +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: redis-master + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 35m + memory: 262144k + uncappedTarget: + cpu: 35m + memory: 262144k + upperBound: + cpu: 39m + memory: 262144k + - containerName: server2 + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 33m + memory: 262144k + uncappedTarget: + cpu: 35m + memory: 262144k + upperBound: + cpu: 40m + memory: 262145k` + + vpa, err := decodeVPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if len(vpa.Recomendations) != 2 { + t.Errorf("VPA object must have two recomendations") + } + + recomendation := vpa.Recomendations[0] + expectedContainerName := "server" + if recomendation.ContainerName != expectedContainerName { + t.Errorf("Expected Container Name %+v, got %+v", expectedContainerName, recomendation.ContainerName) + } + expectedCPU := float32(0.035) + expectedMemory := int64(262144000) + if recomendation.Target.CPU != expectedCPU { + t.Errorf("Expected Target Recomended CPU %+v, got %+v", expectedCPU, recomendation.Target.CPU) + } + if recomendation.Target.Memory != expectedMemory { + t.Errorf("Expected Target Recomended Memory %+v, got %+v", expectedMemory, recomendation.Target.Memory) + } + expectedCPU = float32(0.039) + expectedMemory = int64(262144000) + if recomendation.UpperBound.CPU != expectedCPU { + t.Errorf("Expected UpperBound Recomended CPU %+v, got %+v", expectedCPU, recomendation.UpperBound.CPU) + } + if recomendation.UpperBound.Memory != expectedMemory { + t.Errorf("Expected UpperBound Recomended Memory %+v, got %+v", expectedMemory, recomendation.UpperBound.Memory) + } + + recomendation = vpa.Recomendations[1] + expectedContainerName = "server2" + if recomendation.ContainerName != expectedContainerName { + t.Errorf("Expected Container Name %+v, got %+v", expectedContainerName, recomendation.ContainerName) + } + expectedCPU = float32(0.033) + expectedMemory = int64(262144000) + if recomendation.Target.CPU != expectedCPU { + t.Errorf("Expected Target Recomended CPU %+v, got %+v", expectedCPU, recomendation.Target.CPU) + } + if recomendation.Target.Memory != expectedMemory { + t.Errorf("Expected Target Recomended Memory %+v, got %+v", expectedMemory, recomendation.Target.Memory) + } + expectedCPU = float32(0.040) + expectedMemory = int64(262145000) + if recomendation.UpperBound.CPU != expectedCPU { + t.Errorf("Expected UpperBound Recomended CPU %+v, got %+v", expectedCPU, recomendation.UpperBound.CPU) + } + if recomendation.UpperBound.Memory != expectedMemory { + t.Errorf("Expected UpperBound Recomended Memory %+v, got %+v", expectedMemory, recomendation.UpperBound.Memory) + } +} + +func TestVPAV1beta2WithStatus(t *testing.T) { + yaml := ` +apiVersion: autoscaling.k8s.io/v1beta2 +kind: VerticalPodAutoscaler +metadata: + name: redis-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: redis-master + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 35m + memory: 262144k + uncappedTarget: + cpu: 35m + memory: 262144k + upperBound: + cpu: 39m + memory: 262144k` + + vpa, err := decodeVPA([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + expectedNS := "default" + if got := vpa.Namespace; got != expectedNS { + t.Errorf("Expected Namespace %+v, got %+v", expectedNS, got) + } + + expectedName := "redis-vpa" + if got := vpa.Name; got != expectedName { + t.Errorf("Expected Name %+v, got %+v", expectedName, got) + } + + expectedTargetRef := TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "redis-master"} + if got := vpa.TargetRef; got != expectedTargetRef { + t.Errorf("Expected TargetRef %+v, got %+v", expectedTargetRef, got) + } + + if !vpa.IsInRecomendationMode { + t.Error("VPA object is in recomendation mode") + } + + recomendation := vpa.Recomendations[0] + expectedCPU := float32(0.035) + expectedMemory := int64(262144000) + if recomendation.Target.CPU != expectedCPU { + t.Errorf("Expected Target Recomended CPU %+v, got %+v", expectedCPU, recomendation.Target.CPU) + } + if recomendation.Target.Memory != expectedMemory { + t.Errorf("Expected Target Recomended Memory %+v, got %+v", expectedMemory, recomendation.Target.Memory) + } + expectedCPU = float32(0.039) + expectedMemory = int64(262144000) + if recomendation.UpperBound.CPU != expectedCPU { + t.Errorf("Expected UpperBound Recomended CPU %+v, got %+v", expectedCPU, recomendation.UpperBound.CPU) + } + if recomendation.Target.Memory != expectedMemory { + t.Errorf("Expected UpperBound Recomended Memory %+v, got %+v", expectedMemory, recomendation.UpperBound.Memory) + } +} + +func TestVPAList(t *testing.T) { + yaml := ` + apiVersion: v1 + items: + - apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + name: adservice-vpa + namespace: default + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: adservice + namespace: default + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: "351112894" + target: + cpu: 35m + memory: "351198544" + uncappedTarget: + cpu: 35m + memory: "351198544" + upperBound: + cpu: 39m + memory: "394031262" + - apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + name: cartservice-vpa + namespace: default + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: cartservice + namespace: default + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 25m + memory: 262144k + uncappedTarget: + cpu: 25m + memory: 262144k + upperBound: + cpu: 25m + memory: 262144k + kind: List + metadata: + resourceVersion: "" + selfLink: ""` + + vpaList, err := DecodeVPAList([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if len(vpaList) != 2 { + t.Error("VPA List has two objects") + } +} + +func TestVPAListWithUnsuportedVersion(t *testing.T) { + yaml := ` + apiVersion: v1 + items: + - apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + name: adservice-vpa + namespace: default + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: adservice + namespace: default + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: "351112894" + target: + cpu: 35m + memory: "351198544" + uncappedTarget: + cpu: 35m + memory: "351198544" + upperBound: + cpu: 39m + memory: "394031262" + - apiVersion: autoscaling.k8s.io/v1beta1 + kind: VerticalPodAutoscaler + metadata: + name: cartservice-vpa + namespace: default + spec: + selector: + matchLabels: + label: vpa-label + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-02-10T14:20:46Z" + message: Fetching history complete + status: "False" + type: FetchingHistory + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-02-10T14:19:46Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: 262144k + target: + cpu: 25m + memory: 262144k + uncappedTarget: + cpu: 25m + memory: 262144k + upperBound: + cpu: 25m + memory: 262144k + kind: List + metadata: + resourceVersion: "" + selfLink: ""` + + vpaList, err := DecodeVPAList([]byte(yaml)) + if err != nil { + t.Error(err) + return + } + + if len(vpaList) != 1 { + t.Error("VPA List has only one supported objects") + } +} + +func TestVPADecodeListV1(t *testing.T) { + data, err := ioutil.ReadFile("./testdata/vpa-v1.yaml") + if err != nil { + t.Errorf("Error reading file: %+v", err) + } + + vpas, err := DecodeVPAList(data) + if err != nil { + t.Errorf("Error decoding file: %+v", err) + } + noRecommendations := 0 + for _, vpa := range vpas { + if len(vpa.Recomendations) == 0 || + vpa.Recomendations[0].Target.CPU <= 0 && vpa.Recomendations[0].Target.Memory <= 0 { + noRecommendations++ + } + } + if noRecommendations != 8 { + t.Errorf("VPA should have target recomendations for all but 8") + } +} diff --git a/gke-vpa-recommendations/scripts/apis/k8s/decoder.go b/gke-vpa-recommendations/scripts/apis/k8s/decoder.go new file mode 100644 index 0000000000..81b31a2edb --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/decoder.go @@ -0,0 +1,84 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + vpaV1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpaV1beta1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta1" + + hpaV1 "k8s.io/api/autoscaling/v1" + hpaV2beta1 "k8s.io/api/autoscaling/v2beta1" + hpaV2beta2 "k8s.io/api/autoscaling/v2beta2" +) + +var gvkList schema.GroupVersionKind = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "List"} +var gvkVPAV1 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling.k8s.io", Version: "v1", Kind: VPAKind} +var gvkVPAV1beta1 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling.k8s.io", Version: "v1beta1", Kind: VPAKind} +var gvkVPAV1beta2 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling.k8s.io", Version: "v1beta2", Kind: VPAKind} +var gvkHPAV1 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling", Version: "v1", Kind: HPAKind} +var gvkHPAV2beta1 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling", Version: "v2beta1", Kind: HPAKind} +var gvkHPAV2beta2 schema.GroupVersionKind = schema.GroupVersionKind{Group: "autoscaling", Version: "v2beta2", Kind: HPAKind} + +func decode(scheme *runtime.Scheme, data []byte) (runtime.Object, GroupVersionKind, error) { + decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() + obj, gvk, err := decoder.Decode(data, nil, nil) + if err != nil { + return (runtime.Object)(nil), GroupVersionKind{}, err + } + groupVersionKind := GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + } + return obj, groupVersionKind, err +} + +func buildVPAScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gvkList, &vpaV1.VerticalPodAutoscalerList{}) + scheme.AddKnownTypeWithName(gvkVPAV1, &vpaV1.VerticalPodAutoscaler{}) + scheme.AddKnownTypeWithName(gvkVPAV1beta1, &vpaV1beta1.VerticalPodAutoscaler{}) + scheme.AddKnownTypeWithName(gvkVPAV1beta2, &vpaV1.VerticalPodAutoscaler{}) + return scheme +} + +func buildHPAListV1Scheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gvkList, &hpaV1.HorizontalPodAutoscalerList{}) + return scheme +} + +func buildHPAListV2Beta1Scheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gvkList, &hpaV2beta1.HorizontalPodAutoscalerList{}) + return scheme +} + +func buildHPAListV2Beta2Scheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gvkList, &hpaV2beta2.HorizontalPodAutoscalerList{}) + return scheme +} + +func buildHPAScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gvkHPAV1, &hpaV1.HorizontalPodAutoscaler{}) + scheme.AddKnownTypeWithName(gvkHPAV2beta1, &hpaV2beta1.HorizontalPodAutoscaler{}) + scheme.AddKnownTypeWithName(gvkHPAV2beta2, &hpaV2beta2.HorizontalPodAutoscaler{}) + return scheme +} diff --git a/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v1.yaml b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v1.yaml new file mode 100644 index 0000000000..501b612db0 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v1.yaml @@ -0,0 +1,346 @@ +apiVersion: v1 +items: +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:01Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from memory resource + utilization (percentage of request)"},{"type":"ScalingLimited","status":"False","lastTransitionTime":"2021-04-05T19:54:16Z","reason":"DesiredWithinRange","message":"the + desired count is within the acceptable range"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"memory","currentAverageUtilization":48,"currentAverageValue":"32679936"}}]' + autoscaling.alpha.kubernetes.io/metrics: '[{"type":"Resource","resource":{"name":"memory","targetAverageUtilization":60}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"cartservice-memory-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"memory","target":{"averageUtilization":60,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"cartservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: cartservice-memory-hpa + namespace: default + resourceVersion: "508648" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/cartservice-memory-hpa + uid: 5facf481-5435-4a2d-890b-e684c45a2e9f + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: cartservice + status: + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:44Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-06T09:34:46Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:01Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":5,"currentAverageValue":"5m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"checkoutservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"checkoutservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: checkoutservice-cpu-hpa + namespace: default + resourceVersion: "508645" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/checkoutservice-cpu-hpa + uid: 9464f32f-6811-449b-a80c-cecdc0d77285 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: checkoutservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 5 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:44Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:30Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:11:02Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":12,"currentAverageValue":"12m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"currencyservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"currencyservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: currencyservice-cpu-hpa + namespace: default + resourceVersion: "508163" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/currencyservice-cpu-hpa + uid: 3523c873-8784-49fe-92a5-f2f024148abc + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: currencyservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 12 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:44Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:53:59Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:32:03Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":9,"currentAverageValue":"9m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"emailservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"emailservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: emailservice-cpu-hpa + namespace: default + resourceVersion: "508422" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/emailservice-cpu-hpa + uid: 5b1bcbe5-3987-43dc-9eba-207e8b07c887 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: emailservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 9 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:44Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:00Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:00Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":18,"currentAverageValue":"18m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"frontend-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"frontend"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: frontend-cpu-hpa + namespace: default + resourceVersion: "508649" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/frontend-cpu-hpa + uid: 671867b0-9877-426d-afd0-073dd98dad4c + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 18 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:55:02Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:30:29Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":4,"currentAverageValue":"12m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"loadgenerator-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"loadgenerator"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: loadgenerator-cpu-hpa + namespace: default + resourceVersion: "508650" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/loadgenerator-cpu-hpa + uid: 760756c4-1499-4f39-9e37-f409f4b6d82b + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: loadgenerator + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 4 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":5,"currentAverageValue":"5m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"paymentservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"paymentservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: paymentservice-cpu-hpa + namespace: default + resourceVersion: "508653" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/paymentservice-cpu-hpa + uid: 054713f6-de22-4553-82c4-f11938133a54 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: paymentservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 5 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:01Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":11,"currentAverageValue":"11m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"productcatalogservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"productcatalogservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: productcatalogservice-cpu-hpa + namespace: default + resourceVersion: "508646" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/productcatalogservice-cpu-hpa + uid: 6232b8b4-269d-456f-bdff-09abb337c6fc + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: productcatalogservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 11 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:01Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":17,"currentAverageValue":"17m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"recommendationservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"recommendationservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: recommendationservice-cpu-hpa + namespace: default + resourceVersion: "508424" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/recommendationservice-cpu-hpa + uid: 422f13f1-dee0-48b4-855a-de4022783794 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: recommendationservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 17 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:01Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":7,"currentAverageValue":"5m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"redis-cart-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"redis-cart"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: redis-cart-cpu-hpa + namespace: default + resourceVersion: "508426" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/redis-cart-cpu-hpa + uid: 763c8691-240a-4286-8213-61a02e0afa17 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: redis-cart + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 7 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + autoscaling.alpha.kubernetes.io/conditions: '[{"type":"AbleToScale","status":"True","lastTransitionTime":"2021-04-05T19:53:45Z","reason":"ReadyForNewScale","message":"recommended + size matches current size"},{"type":"ScalingActive","status":"True","lastTransitionTime":"2021-04-05T19:54:31Z","reason":"ValidMetricFound","message":"the + HPA was able to successfully calculate a replica count from cpu resource utilization + (percentage of request)"},{"type":"ScalingLimited","status":"True","lastTransitionTime":"2021-04-06T12:31:32Z","reason":"TooFewReplicas","message":"the + desired replica count is increasing faster than the maximum scale rate"}]' + autoscaling.alpha.kubernetes.io/current-metrics: '[{"type":"Resource","resource":{"name":"cpu","currentAverageUtilization":5,"currentAverageValue":"5m"}}]' + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"shippingservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"shippingservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: shippingservice-cpu-hpa + namespace: default + resourceVersion: "508647" + selfLink: /apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/shippingservice-cpu-hpa + uid: 867b80f5-ca15-41a6-a9df-0e5ced3028b1 + spec: + maxReplicas: 100 + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: shippingservice + targetCPUUtilizationPercentage: 70 + status: + currentCPUUtilizationPercentage: 5 + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +kind: List +metadata: + resourceVersion: "" + selfLink: "" diff --git a/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta1.yaml b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta1.yaml new file mode 100644 index 0000000000..ddb807a5c5 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta1.yaml @@ -0,0 +1,577 @@ +apiVersion: v1 +items: +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"cartservice-memory-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"memory","target":{"averageUtilization":60,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"cartservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: cartservice-memory-hpa + namespace: default + resourceVersion: "508428" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/cartservice-memory-hpa + uid: 5facf481-5435-4a2d-890b-e684c45a2e9f + spec: + maxReplicas: 100 + metrics: + - resource: + name: memory + targetAverageUtilization: 60 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: cartservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:01Z" + message: the HPA was able to successfully calculate a replica count from memory + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-05T19:54:16Z" + message: the desired count is within the acceptable range + reason: DesiredWithinRange + status: "False" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 48 + currentAverageValue: "32681984" + name: memory + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"checkoutservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"checkoutservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: checkoutservice-cpu-hpa + namespace: default + resourceVersion: "508154" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/checkoutservice-cpu-hpa + uid: 9464f32f-6811-449b-a80c-cecdc0d77285 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: checkoutservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-06T09:34:46Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 4 + currentAverageValue: 4m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"currencyservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"currencyservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: currencyservice-cpu-hpa + namespace: default + resourceVersion: "508163" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/currencyservice-cpu-hpa + uid: 3523c873-8784-49fe-92a5-f2f024148abc + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: currencyservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:30Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:11:02Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 12 + currentAverageValue: 12m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"emailservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"emailservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: emailservice-cpu-hpa + namespace: default + resourceVersion: "508422" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/emailservice-cpu-hpa + uid: 5b1bcbe5-3987-43dc-9eba-207e8b07c887 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: emailservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:53:59Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:32:03Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 9 + currentAverageValue: 9m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"frontend-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"frontend"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: frontend-cpu-hpa + namespace: default + resourceVersion: "508431" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/frontend-cpu-hpa + uid: 671867b0-9877-426d-afd0-073dd98dad4c + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:00Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:00Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 22 + currentAverageValue: 22m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"loadgenerator-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"loadgenerator"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: loadgenerator-cpu-hpa + namespace: default + resourceVersion: "508432" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/loadgenerator-cpu-hpa + uid: 760756c4-1499-4f39-9e37-f409f4b6d82b + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: loadgenerator + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:55:02Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:30:29Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 3 + currentAverageValue: 11m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"paymentservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"paymentservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: paymentservice-cpu-hpa + namespace: default + resourceVersion: "508433" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/paymentservice-cpu-hpa + uid: 054713f6-de22-4553-82c4-f11938133a54 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: paymentservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 4 + currentAverageValue: 4m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"productcatalogservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"productcatalogservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: productcatalogservice-cpu-hpa + namespace: default + resourceVersion: "508423" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/productcatalogservice-cpu-hpa + uid: 6232b8b4-269d-456f-bdff-09abb337c6fc + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: productcatalogservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 15 + currentAverageValue: 15m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"recommendationservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"recommendationservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: recommendationservice-cpu-hpa + namespace: default + resourceVersion: "508424" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/recommendationservice-cpu-hpa + uid: 422f13f1-dee0-48b4-855a-de4022783794 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: recommendationservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 17 + currentAverageValue: 17m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"redis-cart-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"redis-cart"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: redis-cart-cpu-hpa + namespace: default + resourceVersion: "508426" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/redis-cart-cpu-hpa + uid: 763c8691-240a-4286-8213-61a02e0afa17 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: redis-cart + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 7 + currentAverageValue: 5m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"shippingservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"shippingservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: shippingservice-cpu-hpa + namespace: default + resourceVersion: "508159" + selfLink: /apis/autoscaling/v2beta1/namespaces/default/horizontalpodautoscalers/shippingservice-cpu-hpa + uid: 867b80f5-ca15-41a6-a9df-0e5ced3028b1 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + targetAverageUtilization: 70 + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: shippingservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:32Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + currentAverageUtilization: 6 + currentAverageValue: 6m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +kind: List +metadata: + resourceVersion: "" + selfLink: "" diff --git a/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta2.yaml b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta2.yaml new file mode 100644 index 0000000000..fbebaf8848 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/testdata/hpa-v2beta2.yaml @@ -0,0 +1,610 @@ +apiVersion: v1 +items: +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"cartservice-memory-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"memory","target":{"averageUtilization":60,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"cartservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: cartservice-memory-hpa + namespace: default + resourceVersion: "497583" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/cartservice-memory-hpa + uid: 5facf481-5435-4a2d-890b-e684c45a2e9f + spec: + maxReplicas: 100 + metrics: + - resource: + name: memory + target: + averageUtilization: 60 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: cartservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:01Z" + message: the HPA was able to successfully calculate a replica count from memory + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-05T19:54:16Z" + message: the desired count is within the acceptable range + reason: DesiredWithinRange + status: "False" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 48 + averageValue: "32524288" + name: memory + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"checkoutservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"checkoutservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: checkoutservice-cpu-hpa + namespace: default + resourceVersion: "497687" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/checkoutservice-cpu-hpa + uid: 9464f32f-6811-449b-a80c-cecdc0d77285 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: checkoutservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-06T09:34:46Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 5 + averageValue: 5m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"currencyservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"currencyservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: currencyservice-cpu-hpa + namespace: default + resourceVersion: "497584" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/currencyservice-cpu-hpa + uid: 3523c873-8784-49fe-92a5-f2f024148abc + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: currencyservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:30Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:11:02Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 14 + averageValue: 14m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"emailservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"emailservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: emailservice-cpu-hpa + namespace: default + resourceVersion: "497581" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/emailservice-cpu-hpa + uid: 5b1bcbe5-3987-43dc-9eba-207e8b07c887 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: emailservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:53:59Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:32:03Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 10 + averageValue: 10m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:44Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"frontend-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"frontend"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: frontend-cpu-hpa + namespace: default + resourceVersion: "497585" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/frontend-cpu-hpa + uid: 671867b0-9877-426d-afd0-073dd98dad4c + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:44Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:00Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:00Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 20 + averageValue: 20m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"loadgenerator-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"loadgenerator"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: loadgenerator-cpu-hpa + namespace: default + resourceVersion: "497685" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/loadgenerator-cpu-hpa + uid: 760756c4-1499-4f39-9e37-f409f4b6d82b + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: loadgenerator + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:55:02Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:30:29Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 4 + averageValue: 12m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"paymentservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"paymentservice"}}} + creationTimestamp: "2021-04-05T19:53:28Z" + name: paymentservice-cpu-hpa + namespace: default + resourceVersion: "497577" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/paymentservice-cpu-hpa + uid: 054713f6-de22-4553-82c4-f11938133a54 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: paymentservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 4 + averageValue: 4m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"productcatalogservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"productcatalogservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: productcatalogservice-cpu-hpa + namespace: default + resourceVersion: "497582" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/productcatalogservice-cpu-hpa + uid: 6232b8b4-269d-456f-bdff-09abb337c6fc + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: productcatalogservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 11 + averageValue: 11m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"recommendationservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"recommendationservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: recommendationservice-cpu-hpa + namespace: default + resourceVersion: "497578" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/recommendationservice-cpu-hpa + uid: 422f13f1-dee0-48b4-855a-de4022783794 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: recommendationservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 17 + averageValue: 17m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"redis-cart-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"redis-cart"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: redis-cart-cpu-hpa + namespace: default + resourceVersion: "481082" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/redis-cart-cpu-hpa + uid: 763c8691-240a-4286-8213-61a02e0afa17 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: redis-cart + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:01Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 7 + averageValue: 5m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +- apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling/v2beta2","kind":"HorizontalPodAutoscaler","metadata":{"annotations":{},"name":"shippingservice-cpu-hpa","namespace":"default"},"spec":{"maxReplicas":100,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":70,"type":"Utilization"}},"type":"Resource"}],"minReplicas":2,"scaleTargetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"shippingservice"}}} + creationTimestamp: "2021-04-05T19:53:29Z" + name: shippingservice-cpu-hpa + namespace: default + resourceVersion: "497352" + selfLink: /apis/autoscaling/v2beta2/namespaces/default/horizontalpodautoscalers/shippingservice-cpu-hpa + uid: 867b80f5-ca15-41a6-a9df-0e5ced3028b1 + spec: + maxReplicas: 100 + metrics: + - resource: + name: cpu + target: + averageUtilization: 70 + type: Utilization + type: Resource + minReplicas: 2 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: shippingservice + status: + conditions: + - lastTransitionTime: "2021-04-05T19:53:45Z" + message: recommended size matches current size + reason: ReadyForNewScale + status: "True" + type: AbleToScale + - lastTransitionTime: "2021-04-05T19:54:31Z" + message: the HPA was able to successfully calculate a replica count from cpu + resource utilization (percentage of request) + reason: ValidMetricFound + status: "True" + type: ScalingActive + - lastTransitionTime: "2021-04-06T12:31:32Z" + message: the desired replica count is increasing faster than the maximum scale + rate + reason: TooFewReplicas + status: "True" + type: ScalingLimited + currentMetrics: + - resource: + current: + averageUtilization: 7 + averageValue: 7m + name: cpu + type: Resource + currentReplicas: 2 + desiredReplicas: 2 + lastScaleTime: "2021-04-05T19:53:45Z" +kind: List +metadata: + resourceVersion: "" + selfLink: "" diff --git a/gke-vpa-recommendations/scripts/apis/k8s/testdata/vpa-v1.yaml b/gke-vpa-recommendations/scripts/apis/k8s/testdata/vpa-v1.yaml new file mode 100644 index 0000000000..8282a63f7e --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/testdata/vpa-v1.yaml @@ -0,0 +1,1260 @@ +apiVersion: v1 +items: +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"adservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"adservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 1016 + name: adservice-vpa + namespace: default + resourceVersion: "528634" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/adservice-vpa + uid: 89ceaa6b-abb4-4ab0-91d9-1c779ea58899 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: adservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-06T12:37:09Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-06T12:28:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 40m + memory: "181403648" + target: + cpu: 55m + memory: "186646528" + uncappedTarget: + cpu: 55m + memory: "186646528" + upperBound: + cpu: 850m + memory: "2767192064" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"cartservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"cartservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 439 + name: cartservice-vpa + namespace: default + resourceVersion: "521536" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/cartservice-vpa + uid: 052c799b-aa07-420b-8a06-5b1fe771b9f6 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: cartservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 15m + memory: "39845888" + target: + cpu: 20m + memory: "42991616" + uncappedTarget: + cpu: 20m + memory: "42991616" + upperBound: + cpu: 50m + memory: "98566144" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"checkoutservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"checkoutservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 283 + name: checkoutservice-vpa + namespace: default + resourceVersion: "514854" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/checkoutservice-vpa + uid: 8d0107c0-4c49-4c78-8faa-f9beb60602c9 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: checkoutservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:59:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 7m + memory: "11534336" + target: + cpu: 20m + memory: "13631488" + uncappedTarget: + cpu: 20m + memory: "13631488" + upperBound: + cpu: 50m + memory: "31457280" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"currencyservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"currencyservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 851 + name: currencyservice-vpa + namespace: default + resourceVersion: "527697" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/currencyservice-vpa + uid: da5fae56-d0c5-45dd-95d7-e5d1f92187c9 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: currencyservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:59:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 15m + memory: 262144k + target: + cpu: 30m + memory: 262144k + uncappedTarget: + cpu: 30m + memory: 262144k + upperBound: + cpu: 75m + memory: "607125504" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"emailservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"emailservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 471 + name: emailservice-vpa + namespace: default + resourceVersion: "523413" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/emailservice-vpa + uid: 149f64f2-e898-4596-b285-d1d99cb3d215 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: emailservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:59:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 9m + memory: "48234496" + target: + cpu: 20m + memory: "50331648" + uncappedTarget: + cpu: 20m + memory: "50331648" + upperBound: + cpu: 40m + memory: "116391936" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"frontend-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"frontend"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 311 + name: frontend-vpa + namespace: default + resourceVersion: "515376" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/frontend-vpa + uid: 0fc307d2-0c99-4415-aeff-3a97b7b29268 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: frontend + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:59:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: "13631488" + target: + cpu: 45m + memory: "15728640" + uncappedTarget: + cpu: 45m + memory: "15728640" + upperBound: + cpu: 115m + memory: "34603008" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"loadgenerator-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"loadgenerator"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 471 + name: loadgenerator-vpa + namespace: default + resourceVersion: "522478" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/loadgenerator-vpa + uid: 633947cc-f0ad-41ee-bfaa-0c71f598dada + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: loadgenerator + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: main + lowerBound: + cpu: 15m + memory: "44040192" + target: + cpu: 25m + memory: "60817408" + uncappedTarget: + cpu: 25m + memory: "60817408" + upperBound: + cpu: 55m + memory: "139460608" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"paymentservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"paymentservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 534 + name: paymentservice-vpa + namespace: default + resourceVersion: "523880" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/paymentservice-vpa + uid: 90f570aa-55c7-4039-8486-c9e1d61df868 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: paymentservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 6m + memory: "77594624" + target: + cpu: 10m + memory: "81788928" + uncappedTarget: + cpu: 10m + memory: "81788928" + upperBound: + cpu: 25m + memory: "187695104" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"productcatalogservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"productcatalogservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 290 + name: productcatalogservice-vpa + namespace: default + resourceVersion: "528636" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/productcatalogservice-vpa + uid: c4dbcd52-e3be-4aae-87a0-94461a5f18cd + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: productcatalogservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 15m + memory: "12582912" + target: + cpu: 35m + memory: "13631488" + uncappedTarget: + cpu: 35m + memory: "13631488" + upperBound: + cpu: 75m + memory: "30408704" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"recommendationservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"recommendationservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 799 + name: recommendationservice-vpa + namespace: default + resourceVersion: "528629" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/recommendationservice-vpa + uid: 37db6e0c-2a8a-4467-a924-f44b67dce03a + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: recommendationservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 25m + memory: "78643200" + target: + cpu: 35m + memory: "660602880" + uncappedTarget: + cpu: 35m + memory: "660602880" + upperBound: + cpu: 80m + memory: "1527775232" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"redis-cart-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"redis-cart"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 151 + name: redis-cart-vpa + namespace: default + resourceVersion: "485422" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/redis-cart-vpa + uid: 11ea66a8-292e-4354-99c6-52669524543d + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: redis-cart + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: redis + lowerBound: + cpu: 3m + memory: "3145728" + target: + cpu: 6m + memory: "4194304" + uncappedTarget: + cpu: 6m + memory: "4194304" + upperBound: + cpu: 15m + memory: "8388608" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"shippingservice-vpa","namespace":"default"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"shippingservice"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:33Z" + generation: 262 + name: shippingservice-vpa + namespace: default + resourceVersion: "525366" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/default/verticalpodautoscalers/shippingservice-vpa + uid: a8497c25-d5f9-42f7-b69f-c7bcc8ed01c6 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: shippingservice + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:00:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: server + lowerBound: + cpu: 5m + memory: "11534336" + target: + cpu: 10m + memory: "13631488" + uncappedTarget: + cpu: 10m + memory: "13631488" + upperBound: + cpu: 25m + memory: "29360128" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"event-exporter-gke-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"event-exporter-gke"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 623 + name: event-exporter-gke-vpa + namespace: kube-system + resourceVersion: "527700" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/event-exporter-gke-vpa + uid: 2cda3f94-f375-4a3c-9786-1b9a27dc0f2a + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: event-exporter-gke + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:04:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: event-exporter + lowerBound: + cpu: 2m + memory: "12582912" + target: + cpu: 4m + memory: "12582912" + uncappedTarget: + cpu: 4m + memory: "12582912" + upperBound: + cpu: 10m + memory: "29360128" + - containerName: prometheus-to-sd-exporter + lowerBound: + cpu: 3m + memory: "8388608" + target: + cpu: 5m + memory: "8388608" + uncappedTarget: + cpu: 5m + memory: "8388608" + upperBound: + cpu: 15m + memory: "18874368" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"fluentbit-gke-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"fluentbit-gke"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 705 + name: fluentbit-gke-vpa + namespace: kube-system + resourceVersion: "528630" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/fluentbit-gke-vpa + uid: fe02ff0b-1697-4ec0-b758-3f2fdb68952f + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: fluentbit-gke + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:58:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: fluentbit + lowerBound: + cpu: 8m + memory: "14680064" + target: + cpu: 20m + memory: "16777216" + uncappedTarget: + cpu: 20m + memory: "16777216" + upperBound: + cpu: 50m + memory: "38797312" + - containerName: fluentbit-gke + lowerBound: + cpu: 5m + memory: "15728640" + target: + cpu: 10m + memory: "16777216" + uncappedTarget: + cpu: 10m + memory: "16777216" + upperBound: + cpu: 30m + memory: "38797312" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"gke-metadata-server-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"gke-metadata-server"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 2 + name: gke-metadata-server-vpa + namespace: kube-system + resourceVersion: "3038" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/gke-metadata-server-vpa + uid: 40193b73-43e0-4576-83f0-632639fe015f + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: gke-metadata-server + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:07Z" + message: 'Cannot read targetRef. Reason: DaemonSet kube-system/gke-metadata-server + does not exist' + status: "True" + type: ConfigUnsupported + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"gke-metrics-agent-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"gke-metrics-agent"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 370 + name: gke-metrics-agent-vpa + namespace: kube-system + resourceVersion: "515844" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/gke-metrics-agent-vpa + uid: 0629f772-ba1c-4179-a83c-53492be46735 + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: gke-metrics-agent + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:58:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: gke-metrics-agent + lowerBound: + cpu: 6m + memory: "27262976" + target: + cpu: 10m + memory: "29360128" + uncappedTarget: + cpu: 10m + memory: "29360128" + upperBound: + cpu: 25m + memory: "67108864" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"gke-metrics-agent-windows-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"gke-metrics-agent-windows"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 2 + name: gke-metrics-agent-windows-vpa + namespace: kube-system + resourceVersion: "3058" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/gke-metrics-agent-windows-vpa + uid: 1a68e407-d4eb-4900-a459-686808ce7033 + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: gke-metrics-agent-windows + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"kube-dns-autoscaler-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"kube-dns-autoscaler"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 179 + name: kube-dns-autoscaler-vpa + namespace: kube-system + resourceVersion: "515377" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/kube-dns-autoscaler-vpa + uid: dbe4e7a0-240d-4e31-bba5-540deaf2c5ed + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: kube-dns-autoscaler + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:04:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: autoscaler + lowerBound: + cpu: 1m + memory: "6291456" + target: + cpu: 1m + memory: "6291456" + uncappedTarget: + cpu: 1m + memory: "6291456" + upperBound: + cpu: 2m + memory: "13631488" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"kube-dns-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"kube-dns"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 894 + name: kube-dns-vpa + namespace: kube-system + resourceVersion: "528628" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/kube-dns-vpa + uid: a22ac463-ee1e-4e07-bc28-2702b0f8a490 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: kube-dns + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:59:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: dnsmasq + lowerBound: + cpu: 1m + memory: "9437184" + target: + cpu: 2m + memory: "9437184" + uncappedTarget: + cpu: 2m + memory: "9437184" + upperBound: + cpu: 5m + memory: "20971520" + - containerName: kubedns + lowerBound: + cpu: 2m + memory: "11534336" + target: + cpu: 3m + memory: "12582912" + uncappedTarget: + cpu: 3m + memory: "12582912" + upperBound: + cpu: 7m + memory: "27262976" + - containerName: prometheus-to-sd + lowerBound: + cpu: 3m + memory: "9437184" + target: + cpu: 6m + memory: "10485760" + uncappedTarget: + cpu: 6m + memory: "10485760" + upperBound: + cpu: 15m + memory: "22020096" + - containerName: sidecar + lowerBound: + cpu: 1m + memory: "14680064" + target: + cpu: 2m + memory: "14680064" + uncappedTarget: + cpu: 2m + memory: "14680064" + upperBound: + cpu: 5m + memory: "33554432" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"kube-proxy-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"kube-proxy"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 2 + name: kube-proxy-vpa + namespace: kube-system + resourceVersion: "3027" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/kube-proxy-vpa + uid: 476f2388-aab7-46f6-9c99-80a96b680ebb + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: kube-proxy + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"l7-default-backend-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"l7-default-backend"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 147 + name: l7-default-backend-vpa + namespace: kube-system + resourceVersion: "509623" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/l7-default-backend-vpa + uid: 846fa87d-3c2c-4e0e-b6f1-9fcbad7ecf42 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: l7-default-backend + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:04:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: default-http-backend + lowerBound: + cpu: 1m + memory: "6291456" + target: + cpu: 1m + memory: "6291456" + uncappedTarget: + cpu: 1m + memory: "6291456" + upperBound: + cpu: 2m + memory: "14680064" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"mcs-core-dns-autoscaler-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"mcs-core-dns-autoscaler"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 2 + name: mcs-core-dns-autoscaler-vpa + namespace: kube-system + resourceVersion: "3031" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/mcs-core-dns-autoscaler-vpa + uid: a0c42056-9c2e-4432-bd13-625bcf848f89 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mcs-core-dns-autoscaler + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:07Z" + message: 'Cannot read targetRef. Reason: Deployment kube-system/mcs-core-dns-autoscaler + does not exist' + status: "True" + type: ConfigUnsupported + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"mcs-core-dns-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"mcs-core-dns"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:30Z" + generation: 2 + name: mcs-core-dns-vpa + namespace: kube-system + resourceVersion: "3030" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/mcs-core-dns-vpa + uid: 974edd18-3c31-4d3d-91f5-cc50e60e4f64 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mcs-core-dns + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:07Z" + message: 'Cannot read targetRef. Reason: Deployment kube-system/mcs-core-dns + does not exist' + status: "True" + type: ConfigUnsupported + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"metadata-proxy-v0.1-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"metadata-proxy-v0.1"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 2 + name: metadata-proxy-v0.1-vpa + namespace: kube-system + resourceVersion: "3028" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/metadata-proxy-v0.1-vpa + uid: 12d69ad2-c619-4a6c-847e-8c221eb333ed + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: metadata-proxy-v0.1 + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"metrics-server-v0.3.6-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"metrics-server-v0.3.6"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 689 + name: metrics-server-v0.3.6-vpa + namespace: kube-system + resourceVersion: "526767" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/metrics-server-v0.3.6-vpa + uid: 5aa9f2cc-8e9c-4a46-8243-6963459ee9a2 + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: metrics-server-v0.3.6 + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T20:04:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: metrics-server + lowerBound: + cpu: 5m + memory: "34603008" + target: + cpu: 9m + memory: "34603008" + uncappedTarget: + cpu: 9m + memory: "34603008" + upperBound: + cpu: 25m + memory: "79691776" + - containerName: metrics-server-nanny + lowerBound: + cpu: 1m + memory: "15728640" + target: + cpu: 1m + memory: "15728640" + uncappedTarget: + cpu: 1m + memory: "15728640" + upperBound: + cpu: 2m + memory: "35651584" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"netd-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"netd"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 2 + name: netd-vpa + namespace: kube-system + resourceVersion: "3056" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/netd-vpa + uid: 635b12b5-720e-482b-b8a6-8ee4430e5bb3 + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: netd + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:07Z" + message: 'Cannot read targetRef. Reason: DaemonSet kube-system/netd does not + exist' + status: "True" + type: ConfigUnsupported + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"nvidia-gpu-device-plugin-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"nvidia-gpu-device-plugin"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 2 + name: nvidia-gpu-device-plugin-vpa + namespace: kube-system + resourceVersion: "3049" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/nvidia-gpu-device-plugin-vpa + uid: 376a99dd-b5fc-42f7-bc7d-020f564188b0 + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: nvidia-gpu-device-plugin + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:56:08Z" + message: 'No recommendation provided from the recommendation engine: No pods + found.' + reason: 'No recommendation provided from the recommendation engine: No pods + found.' + status: "False" + type: RecommendationProvided + recommendation: {} +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"prometheus-to-sd-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"DaemonSet","name":"prometheus-to-sd"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:32Z" + generation: 256 + name: prometheus-to-sd-vpa + namespace: kube-system + resourceVersion: "524889" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/prometheus-to-sd-vpa + uid: a6379aa6-5ea2-4173-b7a8-865e697a3659 + spec: + targetRef: + apiVersion: apps/v1 + kind: DaemonSet + name: prometheus-to-sd + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-05T19:58:08Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-05T19:56:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: prometheus-to-sd + lowerBound: + cpu: 2m + memory: "10485760" + target: + cpu: 4m + memory: "12582912" + uncappedTarget: + cpu: 4m + memory: "12582912" + upperBound: + cpu: 10m + memory: "29360128" +- apiVersion: autoscaling.k8s.io/v1 + kind: VerticalPodAutoscaler + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"stackdriver-metadata-agent-cluster-level-vpa","namespace":"kube-system"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"stackdriver-metadata-agent-cluster-level"},"updatePolicy":{"updateMode":"Off"}}} + creationTimestamp: "2021-04-05T19:55:31Z" + generation: 979 + name: stackdriver-metadata-agent-cluster-level-vpa + namespace: kube-system + resourceVersion: "528631" + selfLink: /apis/autoscaling.k8s.io/v1/namespaces/kube-system/verticalpodautoscalers/stackdriver-metadata-agent-cluster-level-vpa + uid: 8521fde5-27b7-4204-b7fa-e837d6d98d8a + spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: stackdriver-metadata-agent-cluster-level + updatePolicy: + updateMode: "Off" + status: + conditions: + - lastTransitionTime: "2021-04-06T12:37:09Z" + status: "False" + type: LowConfidence + - lastTransitionTime: "2021-04-06T12:28:08Z" + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: metadata-agent + lowerBound: + cpu: 7m + memory: "18874368" + target: + cpu: 10m + memory: "18874368" + uncappedTarget: + cpu: 10m + memory: "18874368" + upperBound: + cpu: 155m + memory: "278921216" + - containerName: metadata-agent-nanny + lowerBound: + cpu: 50m + memory: "19922944" + target: + cpu: 75m + memory: "19922944" + uncappedTarget: + cpu: 75m + memory: "19922944" + upperBound: + cpu: 1175m + memory: "289406976" +kind: List +metadata: + resourceVersion: "" + selfLink: "" diff --git a/gke-vpa-recommendations/scripts/apis/k8s/types.go b/gke-vpa-recommendations/scripts/apis/k8s/types.go new file mode 100644 index 0000000000..2e35e5df79 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/k8s/types.go @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package k8s + +const ( + // VPAKind just to avoid mispeling + VPAKind = "VerticalPodAutoscaler" + // HPAKind just to avoid mispeling + HPAKind = "HorizontalPodAutoscaler" +) + +// GroupVersionKind is the reprsentation of k8s type +// This object is used to to avoid sprawl of dependent library (eg. apimachinary) across the code +// This will allow easy migration to others library (eg. kyaml) in the future once the dependency is all encapulated into k8s_decoder.go +type GroupVersionKind struct { + Group string + Version string + Kind string +} + +//Resource represents the VPA Resource recomendation +type Resource struct { + CPU float32 + Memory int64 +} + +// VPARecomendation represents the VPA Container Recomendations +type VPARecomendation struct { + ContainerName string + Target Resource + UpperBound Resource +} + +// TargetRef references the object which VPA applies to +type TargetRef struct { + APIVersion string + Kind string + Name string +} + +// VPA is the simplified reprsentation of k8s VPA +// Client doesn't need to handle different version and the complexity of k8s.io package +type VPA struct { + Namespace string + Name string + TargetRef TargetRef + IsInRecomendationMode bool + Recomendations []VPARecomendation +} + +// HPA is the simplified reprsentation of k8s HPA +// Client doesn't need to handle different version and the complexity of k8s.io package +type HPA struct { + Namespace string + Name string + TargetRef TargetRef + MinReplicas int32 + MaxReplicas int32 + TargetCPUPercentage int32 + TargetMemoryPercentage int32 +} diff --git a/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries.go b/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries.go new file mode 100644 index 0000000000..0a7a1e55be --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries.go @@ -0,0 +1,121 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mon + +import ( + "fmt" + "metrics-exporter/apis/k8s" + "strconv" + "strings" + + gce "cloud.google.com/go/compute/metadata" + log "github.com/sirupsen/logrus" + "google.golang.org/api/monitoring/v3" +) + +const ( + hpaCPUMetricType = "custom.googleapis.com/podautoscaler/hpa/cpu/target_utilization" + hpaMemoryMetricType = "custom.googleapis.com/podautoscaler/hpa/memory/target_utilization" +) + +// BuildHPATargetUtilizationTimeSeries buid Timeseries objects for HPA target CPU +func BuildHPATargetUtilizationTimeSeries(hpas []k8s.HPA, now string) []*monitoring.TimeSeries { + var hpaMap map[string]k8s.HPA = make(map[string]k8s.HPA) + tsList := []*monitoring.TimeSeries{} + for _, hpa := range hpas { + if hpa.TargetCPUPercentage > 0 || hpa.TargetMemoryPercentage > 0 { + if hpa.TargetCPUPercentage > 0 { + targetKey := fmt.Sprintf("%s|%s|%s|cpu", hpa.TargetRef.Kind, hpa.Namespace, hpa.TargetRef.Name) + if _, found := hpaMap[targetKey]; !found { + hpaMap[targetKey] = hpa + tsList = append(tsList, buildHPACPUTargetUtilization(hpa, now)) + } else { + // Skip HPA object once we alreay had one in the list with the same target object + log.Infof("Skipping HPA cpu '%s.%s' once '%s.%s' was already loaded", + hpa.Namespace, hpa.Name, hpaMap[targetKey].Namespace, hpaMap[targetKey].Name) + } + } + if hpa.TargetMemoryPercentage > 0 { + targetKey := fmt.Sprintf("%s|%s|%s|memory", hpa.TargetRef.Kind, hpa.Namespace, hpa.TargetRef.Name) + if _, found := hpaMap[targetKey]; !found { + hpaMap[targetKey] = hpa + tsList = append(tsList, buildHPAMemoryTargetUtilization(hpa, now)) + } else { + // Skip HPA object once we alreay had one in the list with the same target object + log.Infof("Skipping HPA memory '%s.%s' once '%s.%s' was already loaded", + hpa.Namespace, hpa.Name, hpaMap[targetKey].Namespace, hpaMap[targetKey].Name) + } + } + } else { + log.Infof("Skipping HPA '%s.%s' once it doesn't configure either Target CPU or Target Memory", hpa.Namespace, hpa.Name) + } + } + return tsList +} + +func buildHPACPUTargetUtilization(hpa k8s.HPA, now string) *monitoring.TimeSeries { + metric := hpaCPUMetricType + value := int64(hpa.TargetCPUPercentage) + return buildHPATargetUtilization(metric, value, hpa, now) +} + +func buildHPAMemoryTargetUtilization(hpa k8s.HPA, now string) *monitoring.TimeSeries { + metric := hpaMemoryMetricType + value := int64(hpa.TargetMemoryPercentage) + return buildHPATargetUtilization(metric, value, hpa, now) +} + +func buildHPATargetUtilization(metric string, value int64, hpa k8s.HPA, now string) *monitoring.TimeSeries { + return &monitoring.TimeSeries{ + Resource: &monitoring.MonitoredResource{ + Type: "k8s_pod", + Labels: buildHPAResourceLabels(hpa), + }, + Metric: &monitoring.Metric{ + Type: metric, + Labels: map[string]string{ + "targetef_apiversion": hpa.TargetRef.APIVersion, + "targetref_kind": hpa.TargetRef.Kind, + "targetref_name": hpa.TargetRef.Name, + "minReplicas": strconv.Itoa(int(hpa.MinReplicas)), + "maxReplicas": strconv.Itoa(int(hpa.MaxReplicas)), + "object_name": hpa.Name, + }, + }, + Points: []*monitoring.Point{{ + Interval: &monitoring.TimeInterval{ + EndTime: now, + }, + Value: &monitoring.TypedValue{ + Int64Value: &value, + }, + }}, + } +} + +func buildHPAResourceLabels(hpa k8s.HPA) map[string]string { + projectID, _ := gce.ProjectID() + location, _ := gce.InstanceAttributeValue("cluster-location") + location = strings.TrimSpace(location) + clusterName, _ := gce.InstanceAttributeValue("cluster-name") + clusterName = strings.TrimSpace(clusterName) + return map[string]string{ + "project_id": projectID, + "location": location, + "cluster_name": clusterName, + "namespace_name": hpa.Namespace, + "pod_name": hpa.TargetRef.Name, + } +} diff --git a/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries_test.go b/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries_test.go new file mode 100644 index 0000000000..1b1feef8c0 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/mon/builder_hpa_timeseries_test.go @@ -0,0 +1,207 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mon + +import ( + "metrics-exporter/apis/k8s" + "strconv" + "testing" + "time" +) + +func TestBuildHPACPUTargetUtilization(t *testing.T) { + hpa := k8s.HPA{ + Namespace: "default", + Name: "currencyservice-hpa", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetCPUPercentage: 70, + } + + now := time.Now().Format(time.RFC3339) + ts := buildHPACPUTargetUtilization(hpa, now) + + expected := "currencyservice" + if got := ts.Resource.Labels["pod_name"]; got != expected { + t.Errorf("Expected label %+v, got %+v", expected, got) + } + + expected = hpaCPUMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + + metricLabels := map[string]string{ + "targetef_apiversion": hpa.TargetRef.APIVersion, + "targetref_kind": hpa.TargetRef.Kind, + "targetref_name": hpa.TargetRef.Name, + "minReplicas": strconv.Itoa(int(hpa.MinReplicas)), + "maxReplicas": strconv.Itoa(int(hpa.MaxReplicas)), + "object_name": hpa.Name, + } + for key, expected := range metricLabels { + if got := ts.Metric.Labels[key]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + } + + expected = now + if got := ts.Points[0].Interval.EndTime; got != expected { + t.Errorf("Expected EndTime %+v, got %+v", expected, got) + } + + expectedf := int64(70) + if got := ts.Points[0].Value.Int64Value; *got != expectedf { + t.Errorf("Expected Value %+v, got %+v", expectedf, *got) + } +} + +func TestBuildHPAMemoryTargetUtilization(t *testing.T) { + hpa := k8s.HPA{ + Namespace: "default", + Name: "currencyservice-hpa", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetMemoryPercentage: 60, + } + + now := time.Now().Format(time.RFC3339) + ts := buildHPAMemoryTargetUtilization(hpa, now) + + expected := "currencyservice" + if got := ts.Resource.Labels["pod_name"]; got != expected { + t.Errorf("Expected label %+v, got %+v", expected, got) + } + + expected = hpaMemoryMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + + metricLabels := map[string]string{ + "targetef_apiversion": hpa.TargetRef.APIVersion, + "targetref_kind": hpa.TargetRef.Kind, + "targetref_name": hpa.TargetRef.Name, + "minReplicas": strconv.Itoa(int(hpa.MinReplicas)), + "maxReplicas": strconv.Itoa(int(hpa.MaxReplicas)), + "object_name": hpa.Name, + } + for key, expected := range metricLabels { + if got := ts.Metric.Labels[key]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + } + + expected = now + if got := ts.Points[0].Interval.EndTime; got != expected { + t.Errorf("Expected EndTime %+v, got %+v", expected, got) + } + + expectedf := int64(60) + if got := ts.Points[0].Value.Int64Value; *got != expectedf { + t.Errorf("Expected Value %+v, got %+v", expectedf, *got) + } +} + +func TestBuildHPACPUTargetUtilizationTimeSeries(t *testing.T) { + hpas := []k8s.HPA{ + { + Namespace: "default", + Name: "currencyservice-cpu-hpa", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetCPUPercentage: 70, + }, + { + Namespace: "default", + Name: "currencyservice-cpu-hpa2", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetCPUPercentage: 80, + }, + { + Namespace: "default", + Name: "currencyservice-hpa-wrong", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice2", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetCPUPercentage: 0, + }, + { + Namespace: "default", + Name: "currencyservice-memory-hpa", + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice3", + }, + MinReplicas: 1, + MaxReplicas: 10, + TargetMemoryPercentage: 60, + }, + } + + now := time.Now().Format(time.RFC3339) + tsList := BuildHPATargetUtilizationTimeSeries(hpas, now) + + expectedI := 2 + if got := len(tsList); got != expectedI { + t.Errorf("Expected # %+v, got %+v", expectedI, got) + } + + ts := tsList[0] + expected := hpaCPUMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + expected = "currencyservice-cpu-hpa" + if got := ts.Metric.Labels["object_name"]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + + ts = tsList[1] + expected = hpaMemoryMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + expected = "currencyservice-memory-hpa" + if got := ts.Metric.Labels["object_name"]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } +} diff --git a/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries.go b/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries.go new file mode 100644 index 0000000000..34cd4346be --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries.go @@ -0,0 +1,139 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mon + +import ( + "fmt" + "metrics-exporter/apis/k8s" + "strings" + + log "github.com/sirupsen/logrus" + + gce "cloud.google.com/go/compute/metadata" + "google.golang.org/api/monitoring/v3" +) + +const ( + vpaCPUMetricType = "custom.googleapis.com/podautoscaler/vpa/cpu/target_recommendation" + vpaMemoryMetricType = "custom.googleapis.com/podautoscaler/vpa/memory/target_recommendation" +) + +// BuildVPARecommendationTimeSeries buid Timeseries objects for cpu and memory recommendations +func BuildVPARecommendationTimeSeries(vpas []k8s.VPA, now string) []*monitoring.TimeSeries { + var vpaInRecomendationMode map[string]k8s.VPA = make(map[string]k8s.VPA) + tsList := []*monitoring.TimeSeries{} + for _, vpa := range vpas { + if vpa.IsInRecomendationMode { + targetKey := fmt.Sprintf("%s|%s|%s", vpa.TargetRef.Kind, vpa.Namespace, vpa.TargetRef.Name) + if _, found := vpaInRecomendationMode[targetKey]; !found { + vpaInRecomendationMode[targetKey] = vpa + tsList = append(tsList, buildVPARecomendations(vpa, now)...) + } else { + // Skip VPA object once we alreay had one in recommendation mode for the same target object + log.Infof("Skipping VPA '%s.%s' on recommendation mode once '%s.%s' was already loaded", + vpa.Namespace, vpa.Name, vpaInRecomendationMode[targetKey].Namespace, vpaInRecomendationMode[targetKey].Name) + } + } else { + log.Infof("Skipping VPA '%s.%s' once it is not on recommendation mode", vpa.Namespace, vpa.Name) + } + } + return tsList +} + +func buildVPARecomendations(vpa k8s.VPA, now string) []*monitoring.TimeSeries { + tsList := []*monitoring.TimeSeries{} + for _, rec := range vpa.Recomendations { + resourceLabels := buildVPAResourceLabels(vpa, rec.ContainerName) + // vpa cpu recommendation + cpuTs := buildVPACPURecomendation(vpa, rec, resourceLabels, now) + tsList = append(tsList, &cpuTs) + // vpa memory recommendation + memoryTs := buildVPAMemoryRecomendation(vpa, rec, resourceLabels, now) + tsList = append(tsList, &memoryTs) + } + return tsList +} + +func buildVPACPURecomendation(vpa k8s.VPA, rec k8s.VPARecomendation, resourceLabels map[string]string, now string) monitoring.TimeSeries { + numberOfCores := float64(rec.Target.CPU) + return monitoring.TimeSeries{ + Resource: &monitoring.MonitoredResource{ + Type: "k8s_container", + Labels: resourceLabels, + }, + Metric: &monitoring.Metric{ + Type: vpaCPUMetricType, + Labels: map[string]string{ + "targetef_apiversion": vpa.TargetRef.APIVersion, + "targetref_kind": vpa.TargetRef.Kind, + "targetref_name": vpa.TargetRef.Name, + "object_name": vpa.Name, + }, + }, + Points: []*monitoring.Point{{ + Interval: &monitoring.TimeInterval{ + EndTime: now, + }, + Value: &monitoring.TypedValue{ + DoubleValue: &numberOfCores, + }, + }}, + Unit: "{cpu}", // TODO: understand why it is not being used + } +} + +func buildVPAMemoryRecomendation(vpa k8s.VPA, rec k8s.VPARecomendation, resourceLabels map[string]string, now string) monitoring.TimeSeries { + memoryBytes := rec.Target.Memory + return monitoring.TimeSeries{ + Resource: &monitoring.MonitoredResource{ + Type: "k8s_container", + Labels: resourceLabels, + }, + Metric: &monitoring.Metric{ + Type: vpaMemoryMetricType, + Labels: map[string]string{ + "targetef_apiversion": vpa.TargetRef.APIVersion, + "targetref_kind": vpa.TargetRef.Kind, + "targetref_name": vpa.TargetRef.Name, + "object_name": vpa.Name, + }, + }, + Points: []*monitoring.Point{{ + Interval: &monitoring.TimeInterval{ + EndTime: now, + }, + Value: &monitoring.TypedValue{ + Int64Value: &memoryBytes, + }, + }}, + Unit: "By", // TODO: understand why it is not being used + } +} + +func buildVPAResourceLabels(vpa k8s.VPA, containerName string) map[string]string { + projectID, _ := gce.ProjectID() + location, _ := gce.InstanceAttributeValue("cluster-location") + location = strings.TrimSpace(location) + clusterName, _ := gce.InstanceAttributeValue("cluster-name") + clusterName = strings.TrimSpace(clusterName) + return map[string]string{ + "project_id": projectID, + "location": location, + "cluster_name": clusterName, + "namespace_name": vpa.Namespace, + "pod_name": vpa.TargetRef.Name, + "container_name": containerName, + } +} diff --git a/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries_test.go b/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries_test.go new file mode 100644 index 0000000000..60a93beac1 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/mon/builder_vpa_timeseries_test.go @@ -0,0 +1,260 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mon + +import ( + "metrics-exporter/apis/k8s" + "testing" + "time" +) + +func TestBuildVPACPURecomendation(t *testing.T) { + vpa := k8s.VPA{ + Namespace: "default", + Name: "currencyservice", + IsInRecomendationMode: true, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + } + + labels := map[string]string{"name": "a"} + now := time.Now().Format(time.RFC3339) + values := []float64{float64(400), float64(200)} + + for i, rec := range vpa.Recomendations { + ts := buildVPACPURecomendation(vpa, rec, labels, now) + + expected := labels["name"] + if got := ts.Resource.Labels["name"]; got != expected { + t.Errorf("Expected label %+v, got %+v", expected, got) + } + + expected = vpaCPUMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + + metricLabels := map[string]string{ + "targetef_apiversion": vpa.TargetRef.APIVersion, + "targetref_kind": vpa.TargetRef.Kind, + "targetref_name": vpa.TargetRef.Name, + "object_name": vpa.Name, + } + for key, expected := range metricLabels { + if got := ts.Metric.Labels[key]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + } + + expected = now + if got := ts.Points[0].Interval.EndTime; got != expected { + t.Errorf("Expected EndTime %+v, got %+v", expected, got) + } + + expectedf := values[i] + if got := ts.Points[0].Value.DoubleValue; *got != expectedf { + t.Errorf("Expected Value %+v, got %+v", expectedf, *got) + } + } +} + +func TestBuildVPAMemoryRecomendation(t *testing.T) { + vpa := k8s.VPA{ + Namespace: "default", + Name: "currencyservice", + IsInRecomendationMode: true, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + } + + labels := map[string]string{"name": "a"} + now := time.Now().Format(time.RFC3339) + values := []int64{1024, 500} + + for i, rec := range vpa.Recomendations { + ts := buildVPAMemoryRecomendation(vpa, rec, labels, now) + + expected := labels["name"] + if got := ts.Resource.Labels["name"]; got != expected { + t.Errorf("Expected label %+v, got %+v", expected, got) + } + + expected = vpaMemoryMetricType + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + + metricLabels := map[string]string{ + "targetef_apiversion": vpa.TargetRef.APIVersion, + "targetref_kind": vpa.TargetRef.Kind, + "targetref_name": vpa.TargetRef.Name, + "object_name": vpa.Name, + } + for key, expected := range metricLabels { + if got := ts.Metric.Labels[key]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + } + + expected = now + if got := ts.Points[0].Interval.EndTime; got != expected { + t.Errorf("Expected EndTime %+v, got %+v", expected, got) + } + + expectedf := values[i] + if got := ts.Points[0].Value.Int64Value; *got != expectedf { + t.Errorf("Expected Value %+v, got %+v", expectedf, *got) + } + } +} + +func TestBuildVPARecomendations(t *testing.T) { + vpa := k8s.VPA{ + Namespace: "default", + Name: "currencyservice", + IsInRecomendationMode: true, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + } + + now := time.Now().Format(time.RFC3339) + tsList := buildVPARecomendations(vpa, now) + + expected := 4 + if got := len(tsList); got != expected { + t.Errorf("Expected # %+v, got %+v", expected, got) + } + + metrics := []string{vpaCPUMetricType, vpaMemoryMetricType} + for i, ts := range tsList { + expected := metrics[i%2] + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + } +} + +func TestBuildVPARecommendationTimeSeries(t *testing.T) { + vpas := []k8s.VPA{ + { + Namespace: "default", + Name: "currencyservice-1", + IsInRecomendationMode: true, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + }, + { + Namespace: "default", + Name: "currencyservice-2", + IsInRecomendationMode: true, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "currencyservice", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + }, + { + Namespace: "default", + Name: "other", + IsInRecomendationMode: false, + TargetRef: k8s.TargetRef{ + APIVersion: "v1", + Kind: "Deployment", + Name: "other", + }, + Recomendations: []k8s.VPARecomendation{{ + ContainerName: "server", + Target: k8s.Resource{CPU: 400, Memory: 1024}, + }, { + ContainerName: "sidecar", + Target: k8s.Resource{CPU: 200, Memory: 500}, + }, + }, + }, + } + + now := time.Now().Format(time.RFC3339) + tsList := BuildVPARecommendationTimeSeries(vpas, now) + + expected := 4 + if got := len(tsList); got != expected { + t.Errorf("Expected # %+v, got %+v", expected, got) + } + + metrics := []string{vpaCPUMetricType, vpaMemoryMetricType} + for i, ts := range tsList { + expected := metrics[i%2] + if got := ts.Metric.Type; got != expected { + t.Errorf("Expected Metric %+v, got %+v", expected, got) + } + + expected = "currencyservice-1" + if got := ts.Metric.Labels["object_name"]; got != expected { + t.Errorf("Expected Label %+v, got %+v", expected, got) + } + } +} diff --git a/gke-vpa-recommendations/scripts/apis/mon/cloud_monitoring.go b/gke-vpa-recommendations/scripts/apis/mon/cloud_monitoring.go new file mode 100644 index 0000000000..30451cb392 --- /dev/null +++ b/gke-vpa-recommendations/scripts/apis/mon/cloud_monitoring.go @@ -0,0 +1,104 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mon + +import ( + "context" + "fmt" + "strings" + "time" + + gce "cloud.google.com/go/compute/metadata" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/monitoring/v3" +) + +const ( + maxRetries = 3 + retrialBackOff = 350 +) + +// ExportMetrics Export the timeseries list to cloud monitoring +func ExportMetrics(tsList []*monitoring.TimeSeries) error { + service, err := monitoring.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))) + if err != nil { + return err + } + + targetSize := len(tsList) + if targetSize > 0 { + tsChunks := timeSeriesToChunks(tsList, 200) // break into chunks once API accept only 200 per time + for _, tsChunk := range tsChunks { + log.Infof("Exporting chunk with %v metrics", len(tsChunk)) + err := exportChunk(service, tsChunk, 1) + if err != nil { + log.Errorf(fmt.Sprintf("Failed to export chunk with %v metrics. Cause: %+v", len(tsChunk), err)) + logTimeSeriesList(tsChunk) + } else { + log.Infof("Success exporting chunk with %v metrics", len(tsChunk)) + } + } + + } + return nil +} + +func exportChunk(service *monitoring.Service, tsChunk []*monitoring.TimeSeries, retries int) error { + projectID, _ := gce.ProjectID() + project := "projects/" + projectID + request := &monitoring.CreateTimeSeriesRequest{TimeSeries: tsChunk} + _, err := service.Projects.TimeSeries.Create(project, request).Do() + if err != nil { + log.Warnf(fmt.Sprintf("[Trial %v] Failed to export chunk with %v metrics. Cause: %+v", retries, len(tsChunk), err)) + if retries <= maxRetries && strings.Contains(err.Error(), "Error 500") { + // incremental back off + duration := time.Duration(retrialBackOff*retries) * time.Millisecond + time.Sleep(duration) + return exportChunk(service, tsChunk, retries+1) + } + return err + } + return nil +} + +func timeSeriesToChunks(tsList []*monitoring.TimeSeries, chunkSize int) [][]*monitoring.TimeSeries { + chunks := [][]*monitoring.TimeSeries{} + for len(tsList) > chunkSize { + chunks = append(chunks, tsList[:chunkSize-1]) + tsList = tsList[chunkSize:] + } + if len(tsList) > 0 { + chunks = append(chunks, tsList) + } + return chunks +} + +func logTimeSeriesList(tsList []*monitoring.TimeSeries) { + for tsIndex, v := range tsList { + points := make([]float64, len(v.Points)) + for i, p := range v.Points { + v := *p.Value + if v.DoubleValue != nil { + points[i] = *v.DoubleValue + } else { + points[i] = float64(*v.Int64Value) + } + } + log.Errorf("TimeSeries[%v] => Metric: %s | Resource Labels: %v | Metric Labels: %v | Points: %+v", + tsIndex, v.Metric.Type, v.Resource.Labels, v.Metric.Labels, points) + } +} diff --git a/gke-vpa-recommendations/scripts/bigquery_recommendation_schema.json b/gke-vpa-recommendations/scripts/bigquery_recommendation_schema.json new file mode 100644 index 0000000000..ad54c4e8be --- /dev/null +++ b/gke-vpa-recommendations/scripts/bigquery_recommendation_schema.json @@ -0,0 +1,134 @@ +[ + { + "name": "recommendation_timestamp", + "mode": "Required", + "type": "TIMESTAMP" + }, + { + "name": "location", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "project_id", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "cluster_name", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "controller_name", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "controller_type", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "namespace_name", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "container_count", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "cpu_limit_cores", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "cpu_requested_cores", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "memory_limit_bytes", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "memory_requested_bytes", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "memory_request_max_recommendations", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "mem_qos", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "cpu_qos", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "memory_limit_recommendations", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "cpu_request_recommendations", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "cpu_limit_recommendations", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "cpu_delta", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "mem_delta", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "priority", + "mode": "NULLABLE", + "type": "INTEGER" + }, + { + "name": "mem_provision_status", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "mem_provision_risk", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "cpu_provision_status", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "cpu_provision_risk", + "mode": "NULLABLE", + "type": "STRING" + }, + { + "name": "latest", + "mode": "NULLABLE", + "type": "BOOLEAN" + } + + ] + \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/bigquery_schema.json b/gke-vpa-recommendations/scripts/bigquery_schema.json new file mode 100644 index 0000000000..bc80f17d0c --- /dev/null +++ b/gke-vpa-recommendations/scripts/bigquery_schema.json @@ -0,0 +1,48 @@ +[ + { + "name": "metric_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "location", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "project_id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "cluster_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "controller_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "controller_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "namespace_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "points", + "type": "INTEGER", + "mode": "NULLABLE" + } + , + { + "name": "tstamp", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/cloudbuild.yaml b/gke-vpa-recommendations/scripts/cloudbuild.yaml new file mode 100644 index 0000000000..a71f0423c5 --- /dev/null +++ b/gke-vpa-recommendations/scripts/cloudbuild.yaml @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file and other cloudbuild.yaml files are used to ensure that +# our public Docker images such as gcr.io/google-samples/hello-app:1.0 +# are rebuilt and updated upon changes to the repository. + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - 'build' + - '-t' + - 'us-docker.pkg.dev/google-samples/containers/gke/hpa-metrics-exporter' + - '.' + dir: 'gke-vpa-recommendations/scripts' + +# Push images. +images: + - 'us-docker.pkg.dev/google-samples/containers/gke/hpa-metrics-exporter' diff --git a/gke-vpa-recommendations/scripts/deploy_pipeline.sh b/gke-vpa-recommendations/scripts/deploy_pipeline.sh new file mode 100755 index 0000000000..b89db5c2c4 --- /dev/null +++ b/gke-vpa-recommendations/scripts/deploy_pipeline.sh @@ -0,0 +1,45 @@ + +echo "Create a service account to run the pipeline" +gcloud iam service-accounts create mql-export-metrics \ +--display-name "MQL export metrics SA" \ +--description "Used for the function that export monitoring metrics" + +echo "Assigning IAM roles to the service account..." +gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.viewer" +gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.dataEditor" +gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.dataOwner" +gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.jobUser" +gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/run.invoker" + +echo "Creating the Pub/Sub topic..." +gcloud pubsub topics create metric_export + + +echo "Build image and store the image in Artifact registry..." +gcloud builds submit metrics-exporter --config=metrics-exporter/cloudbuild.yaml --substitutions=_REGION=$REGION + + +echo "Deploy the Cloud Run Job.." +gcloud beta run jobs deploy metric-exporter \ + --image=$REGION-docker.pkg.dev/$PROJECT_ID/main/metric-exporter \ + --set-env-vars=PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python \ + --set-env-vars=PROJECT_ID=$PROJECT_ID \ + --execute-now \ + --memory=1Gi \ + --max-retries=1 \ + --parallelism=0 \ + --service-account=mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com \ + --region=$REGION + +echo "Enable the Cloud Scheduler api.." +gcloud services enable cloudscheduler.googleapis.com + + +gcloud scheduler jobs create http metric-exporter \ + --location $REGION \ + --schedule="0 23 * * *" \ + --uri="https://$REGION-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/$PROJECT_ID/jobs/metric-exporter:run" \ + --http-method POST \ + --oauth-service-account-email "mql-export-metrics@$PROJECT_ID.iam.gserviceaccount.com" + +echo "Deployment complete" diff --git a/gke-vpa-recommendations/scripts/go.mod b/gke-vpa-recommendations/scripts/go.mod new file mode 100644 index 0000000000..af7d34c57f --- /dev/null +++ b/gke-vpa-recommendations/scripts/go.mod @@ -0,0 +1,15 @@ +module metrics-exporter + +go 1.15 + +require ( + cloud.google.com/go v0.76.0 + github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76 // indirect + github.com/sirupsen/logrus v1.7.0 + golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + google.golang.org/api v0.39.0 + k8s.io/api v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/autoscaler/vertical-pod-autoscaler v0.9.2 +) diff --git a/gke-vpa-recommendations/scripts/go.sum b/gke-vpa-recommendations/scripts/go.sum new file mode 100644 index 0000000000..8aee45e8eb --- /dev/null +++ b/gke-vpa-recommendations/scripts/go.sum @@ -0,0 +1,626 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.76.0 h1:Ckw+E/QYZgd/5bpI4wz4h6f+jmpvh9S9uSrKNnbicJI= +cloud.google.com/go v0.76.0/go.mod h1:r9EvIAvLrunusnetGdQ50M/gKui1x3zdGW/VELGkdpw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fernandorubbo/costimator v0.0.0-20210121121042-373627a26983 h1:D4+bKAG5RG4gP8bq2wVYi/LyN5OmK3Opg0IgGioiIzg= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76 h1:JypWNzPMSgH5yL0NvFoAIsDRlKFgL0AsS3GO5bg4Pto= +github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c h1:HiAZXo96zOhVhtFHchj/ojzoxCFiPrp9/j0GtS38V3g= +golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.39.0 h1:zHCTXf0NeDdKTgcSQpT+ZflWAqHsEp1GmdpxW09f3YM= +google.golang.org/api v0.39.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210202153253-cf70463f6119 h1:m9+RjTMas6brUP8DBxSAa/WIPFy7FIhKpvk+9Ppce8E= +google.golang.org/genproto v0.0.0-20210202153253-cf70463f6119/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/autoscaler v0.0.0-20210209174259-bad3157beaba h1:1fuOTBP7ORxTVqzs+xyi9ao2CtGdfMKNojjVq7BOWo8= +k8s.io/autoscaler/vertical-pod-autoscaler v0.9.2 h1:fMFkbjo6ElaL3POE3ctxLuX3aR+bEzpo2q1R5Aq2JhE= +k8s.io/autoscaler/vertical-pod-autoscaler v0.9.2/go.mod h1:PwWTGRRCxefhAezrDbG/tRYSAW7etHjjMPAr8fXKVAA= +k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= +k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/metrics v0.18.3/go.mod h1:TkuJE3ezDZ1ym8pYkZoEzJB7HDiFE7qxl+EmExEBoPA= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/gke-vpa-recommendations/scripts/k8s/templates/cpu-hpa.gtpl b/gke-vpa-recommendations/scripts/k8s/templates/cpu-hpa.gtpl new file mode 100644 index 0000000000..aec75f949b --- /dev/null +++ b/gke-vpa-recommendations/scripts/k8s/templates/cpu-hpa.gtpl @@ -0,0 +1,40 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $size := len .items -}} +{{- range $i, $d := .items -}} +{{if (le $i $size)}} + +--- +{{end}} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{$d.metadata.name}}-cpu-hpa + namespace: {{$d.metadata.namespace}} +spec: + scaleTargetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{$d.metadata.name}} + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 +{{- end -}} \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/k8s/templates/hpa-metrics-exporter.yaml b/gke-vpa-recommendations/scripts/k8s/templates/hpa-metrics-exporter.yaml new file mode 100644 index 0000000000..09ce527bad --- /dev/null +++ b/gke-vpa-recommendations/scripts/k8s/templates/hpa-metrics-exporter.yaml @@ -0,0 +1,76 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-exporter-sa + namespace: custom-metrics + annotations: + iam.gke.io/gcp-service-account: svc-metric-exporter@$PROJECT_ID.iam.gserviceaccount.com + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-exporter-cr +rules: +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-exporter-crb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-exporter-cr +subjects: +- kind: ServiceAccount + name: metrics-exporter-sa + namespace: custom-metrics + +--- + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: metrics-exporter + namespace: custom-metrics +spec: + schedule: "* * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: metrics-exporter-sa + nodeSelector: + iam.gke.io/gke-metadata-server-enabled: "true" + containers: + - name: metrics-exporter + image: $REGION-docker.pkg.dev/$PROJECT_ID/main/hpa-metric-exporter:latest + env: + - name: DISABLEVPA + value: "true" + restartPolicy: OnFailure + backoffLimit: 1 + diff --git a/gke-vpa-recommendations/scripts/k8s/templates/memory-hpa.gtpl b/gke-vpa-recommendations/scripts/k8s/templates/memory-hpa.gtpl new file mode 100644 index 0000000000..cb9b90343b --- /dev/null +++ b/gke-vpa-recommendations/scripts/k8s/templates/memory-hpa.gtpl @@ -0,0 +1,40 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $size := len .items -}} +{{- range $i, $d := .items -}} +{{if (le $i $size)}} + +--- +{{end}} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{$d.metadata.name}}-memory-hpa + namespace: {{$d.metadata.namespace}} +spec: + scaleTargetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{$d.metadata.name}} + minReplicas: 2 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 60 +{{- end -}} diff --git a/gke-vpa-recommendations/scripts/k8s/templates/metrics-exporter.yaml b/gke-vpa-recommendations/scripts/k8s/templates/metrics-exporter.yaml new file mode 100644 index 0000000000..365e39906d --- /dev/null +++ b/gke-vpa-recommendations/scripts/k8s/templates/metrics-exporter.yaml @@ -0,0 +1,75 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-exporter-sa + namespace: custom-metrics + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-exporter-cr +rules: +- apiGroups: + - autoscaling.k8s.io + resources: + - verticalpodautoscalers + verbs: + - list +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-exporter-crb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-exporter-cr +subjects: +- kind: ServiceAccount + name: metrics-exporter-sa + namespace: custom-metrics + +--- + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: metrics-exporter + namespace: custom-metrics +spec: + schedule: "* * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccount: metrics-exporter-sa + containers: + - name: metrics-exporter + image: gcr.io/PROJECT_ID/hpa-metrics-exporter + restartPolicy: OnFailure + backoffLimit: 1 + diff --git a/gke-vpa-recommendations/scripts/k8s/templates/vpa.gtpl b/gke-vpa-recommendations/scripts/k8s/templates/vpa.gtpl new file mode 100644 index 0000000000..859c6c8f70 --- /dev/null +++ b/gke-vpa-recommendations/scripts/k8s/templates/vpa.gtpl @@ -0,0 +1,33 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $size := len .items -}} +{{- range $i, $d := .items -}} +{{if (le $i $size)}} + +--- +{{end}} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{$d.metadata.name}}-vpa + namespace: {{$d.metadata.namespace}} +spec: + targetRef: + apiVersion: "{{$d.apiVersion}}" + kind: {{$d.kind}} + name: {{$d.metadata.name}} + updatePolicy: + updateMode: "Off" +{{- end -}} \ No newline at end of file diff --git a/gke-vpa-recommendations/scripts/main.go b/gke-vpa-recommendations/scripts/main.go new file mode 100644 index 0000000000..edc2781fc0 --- /dev/null +++ b/gke-vpa-recommendations/scripts/main.go @@ -0,0 +1,104 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package main + +import ( + "flag" + "fmt" + "metrics-exporter/apis/k8s" + "metrics-exporter/apis/mon" + "os" + "os/exec" + "time" + + log "github.com/sirupsen/logrus" +) + +const version = "v0.0.1" + +var ( + verbosity = flag.String("v", "info", "[Optional] Verbosity: panic|fatal|error|warn|info|debug|trace") +) + +func init() { + flag.Parse() + + level, err := log.ParseLevel(*verbosity) + exitOnError("Invalid 'verbose' parameter", err) + log.SetFormatter(&log.JSONFormatter{ + DisableTimestamp: true, + FieldMap: log.FieldMap{ + log.FieldKeyLevel: "severity", + }, + }) + log.SetOutput(os.Stdout) + log.SetLevel(level) +} + +func main() { + log.Infof("************** METRICS EXPORTER STARTED (version %s) **************", version) + + now := time.Now().Format(time.RFC3339) + + DISABLEVPA := os.Getenv("DISABLEVPA") + + hpas := retrieveHPAs() + tsList := mon.BuildHPATargetUtilizationTimeSeries(hpas, now) + + if DISABLEVPA != "true" { + vpas := retrieveVPAs() + tsList = append(tsList, mon.BuildVPARecommendationTimeSeries(vpas, now)...) + } + + err := mon.ExportMetrics(tsList) + exitOnError("Failed to instantiate cloud monitoring object", err) + + log.Infof("************** METRICS EXPORTER FINISHED *************") +} + +func retrieveVPAs() []k8s.VPA { + cmd := "kubectl get vpa.v1.autoscaling.k8s.io --all-namespaces -o yaml" + out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + data := string(out) + exitOnError(fmt.Sprintf("Failed to execute command: %s\nRoot Cause: %+v", cmd, data), err) + + vpas, err := k8s.DecodeVPAList([]byte(data)) + exitOnError(fmt.Sprintf("Failed to decode VPA list from command: %s", cmd), err) + return vpas +} + +func retrieveHPAs() []k8s.HPA { + cmd := "kubectl get hpa.v2beta2.autoscaling --all-namespaces -o yaml" + out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + data := string(out) + exitOnError(fmt.Sprintf("Failed to execute command: %s\nRoot Cause: %+v", cmd, data), err) + + hpas, err := k8s.DecodeHPAListV2Beta2([]byte(data)) + exitOnError(fmt.Sprintf("Failed to decode HPA list from command: %s", cmd), err) + return hpas +} + +func exitOnError(message string, err error) { + if err != nil { + exitWithError(message, err) + } +} + +func exitWithError(message string, err error) { + fmt.Printf("\nError: %s\nCause: %+v\n\nSee parameters options below:\n", err, message) + flag.PrintDefaults() + os.Exit(-1) +} diff --git a/gke-vpa-recommendations/scripts/setup.sh b/gke-vpa-recommendations/scripts/setup.sh new file mode 100755 index 0000000000..9e5479e2e7 --- /dev/null +++ b/gke-vpa-recommendations/scripts/setup.sh @@ -0,0 +1,65 @@ +echo "Configuring region and zone" +gcloud config set compute/region $REGION +gcloud config set compute/zone $ZONE + +echo "Create a new IAM service account" +gcloud iam service-accounts create svc-metric-exporter \ + --project=${PROJECT_ID} + +echo "Creating a gke cluster" +gcloud container clusters create online-boutique \ + --project=${PROJECT_ID} --zone=${ZONE} \ + --enable-shielded-nodes \ + --shielded-secure-boot \ + --shielded-integrity-monitoring \ + --service-account=svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com \ + --machine-type=e2-standard-2 --num-nodes=5 \ + --workload-pool=${PROJECT_ID}.svc.id.goog + +sleep 7 & +PID=$! +i=1 +sp="/-\|" +echo -n ' ' +while [ -d /proc/$PID ] +do + printf "\b${sp:i++%${#sp}:1}" +done + +echo "Get credentials for your cluster" +gcloud container clusters get-credentials online-boutique + +echo "Granting roles..." +gcloud projects add-iam-policy-binding ${PROJECT_ID} \ + --member="serviceAccount:svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/monitoring.metricWriter" + +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/compute.viewer" + +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/artifactregistry.reader" + +gcloud projects add-iam-policy-binding $PROJECT_ID \ + --member="serviceAccount:svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/container.nodeServiceAgent" + +gcloud iam service-accounts add-iam-policy-binding svc-metric-exporter@${PROJECT_ID}.iam.gserviceaccount.com \ + --role roles/iam.workloadIdentityUser \ + --member "serviceAccount:${PROJECT_ID}.svc.id.goog[custom-metrics/metrics-exporter-sa]" + + +echo "deploy the onlineshop" +kubectl apply -f k8s/online-shop.yaml + +echo "To simulate a more realistic environment, create an HPA for Online Boutique deployments" +kubectl get deployments --field-selector='metadata.name==adservice' -o go-template-file=scripts/k8s/templates/cpu-hpa.gtpl | kubectl apply -f - +kubectl get deployments --field-selector='metadata.name==redis-cart' -o go-template-file=scripts/k8s/templates/memory-hpa.gtpl | kubectl apply -f - +kubectl get hpa + +kubectl create ns custom-metrics + + +echo "SETUP COMPLETE" \ No newline at end of file diff --git a/terraform/google-cloud-build-triggers.tf b/terraform/google-cloud-build-triggers.tf index 88f5d3f610..7a34291805 100644 --- a/terraform/google-cloud-build-triggers.tf +++ b/terraform/google-cloud-build-triggers.tf @@ -220,3 +220,32 @@ resource "google_cloudbuild_trigger" "workload-metrics" { } } } + +resource "google_cloudbuild_trigger" "metrics-exporter" { + name = "kubernetes-engine-samples-metrics-exporter" + filename = "gke-vpa-recommendations/metrics-exporter/cloudbuild.yaml" + included_files = ["gke-vpa-recommendations/metrics-exporter/**"] + description = local.trigger_description + + github { + owner = "GoogleCloudPlatform" + name = "kubernetes-engine-samples" + push { + branch = "^main$" + } + } +} +resource "google_cloudbuild_trigger" "hpa-metrics-exporter" { + name = "kubernetes-engine-samples-hpa-metrics-exporter" + filename = "gke-vpa-recommendations/scripts/cloudbuild.yaml" + included_files = ["gke-vpa-recommendations/scripts/**"] + description = local.trigger_description + + github { + owner = "GoogleCloudPlatform" + name = "kubernetes-engine-samples" + push { + branch = "^main$" + } + } +}