From 5a0c905e40b9d02673d57ed3f0de98493304b60b Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Mon, 16 Dec 2024 14:40:42 +0100 Subject: [PATCH] feat: changes for using local vpa dependency Signed-off-by: David van der Spek --- multidimensional-pod-autoscaler/go.mod | 23 +- multidimensional-pod-autoscaler/go.sum | 46 ++- .../pkg/admission-controller/certs.go | 79 +++- .../pkg/admission-controller/certs_test.go | 150 +++++++ .../pkg/admission-controller/config.go | 138 ++++++- .../pkg/admission-controller/config_test.go | 373 ++++++++++++++++++ .../pkg/admission-controller/logic/server.go | 2 +- .../pkg/admission-controller/main.go | 66 ++-- .../resource/mpa/handler.go | 3 +- .../resource/mpa/matcher.go | 61 ++- .../resource/mpa/matcher_test.go | 14 +- .../resource/pod/handler.go | 5 +- .../resource/pod/handler_test.go | 5 +- .../pkg/recommender/input/cluster_feeder.go | 25 +- .../recommender/input/cluster_feeder_test.go | 5 +- .../input/metrics/metrics_client.go | 118 ++++++ .../input/metrics/metrics_client_test.go | 46 +++ .../input/metrics/metrics_client_test_util.go | 138 +++++++ .../input/metrics/metrics_source.go | 145 +++++++ .../pkg/recommender/main.go | 242 ++++++++++-- .../pkg/recommender/model/cluster.go | 17 +- .../pkg/recommender/model/cluster_test.go | 41 +- .../routines/capping_post_processor.go | 42 ++ .../routines/cpu_integer_post_processor.go | 89 +++++ .../cpu_integer_post_processor_test.go | 239 +++++++++++ .../routines/recommendation_post_processor.go | 27 ++ .../pkg/recommender/routines/recommender.go | 25 +- .../pkg/target/fetcher.go | 10 +- .../pkg/target/mock/fetcher_mock.go | 40 +- .../eviction/pods_eviction_restriction.go | 7 +- .../pods_eviction_restriction_test.go | 28 +- .../pkg/updater/logic/updater.go | 23 +- .../pkg/updater/logic/updater_test.go | 6 +- .../pkg/updater/main.go | 120 +++++- .../pkg/utils/mpa/api.go | 152 ++++--- .../pkg/utils/test/test_utils.go | 2 +- .../pkg/recommender/logic/recommender.go | 5 + 37 files changed, 2238 insertions(+), 319 deletions(-) create mode 100644 multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go create mode 100644 multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go create mode 100644 multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go diff --git a/multidimensional-pod-autoscaler/go.mod b/multidimensional-pod-autoscaler/go.mod index 79e97e8afe7b..f80579c8ab66 100644 --- a/multidimensional-pod-autoscaler/go.mod +++ b/multidimensional-pod-autoscaler/go.mod @@ -35,7 +35,7 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -65,7 +65,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.61.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -86,20 +86,20 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.30.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/term v0.25.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.26.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect @@ -113,7 +113,7 @@ require ( k8s.io/kms v0.32.0 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect k8s.io/kubelet v0.0.0 // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect @@ -125,6 +125,7 @@ replace ( k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.32.0 k8s.io/apimachinery => k8s.io/apimachinery v0.32.0 k8s.io/apiserver => k8s.io/apiserver v0.32.0 + k8s.io/autoscaler/vertical-pod-autoscaler => ../vertical-pod-autoscaler k8s.io/cli-runtime => k8s.io/cli-runtime v0.32.0 k8s.io/client-go => k8s.io/client-go v0.32.0 k8s.io/cloud-provider => k8s.io/cloud-provider v0.32.0 diff --git a/multidimensional-pod-autoscaler/go.sum b/multidimensional-pod-autoscaler/go.sum index bf75a5d14b4e..40730ab6c9c7 100644 --- a/multidimensional-pod-autoscaler/go.sum +++ b/multidimensional-pod-autoscaler/go.sum @@ -32,8 +32,8 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -137,8 +137,8 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -217,8 +217,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -231,31 +231,31 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -277,8 +277,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -301,8 +301,6 @@ k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= k8s.io/apiserver v0.32.0 h1:VJ89ZvQZ8p1sLeiWdRJpRD6oLozNZD2+qVSLi+ft5Qs= k8s.io/apiserver v0.32.0/go.mod h1:HFh+dM1/BE/Hm4bS4nTXHVfN6Z6tFIZPi649n83b4Ag= -k8s.io/autoscaler/vertical-pod-autoscaler v1.2.1 h1:t5t0Rsn4b7iQfiVlGdWSEnEx8pjrSM96Sn4Dvo1QH/Q= -k8s.io/autoscaler/vertical-pod-autoscaler v1.2.1/go.mod h1:9ywHbt0kTrLyeNGgTNm7WEns34PmBMEr+9bDKTxW6wQ= k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= k8s.io/cloud-provider v0.32.0 h1:QXYJGmwME2q2rprymbmw2GroMChQYc/MWN6l/I4Kgp8= @@ -329,8 +327,8 @@ k8s.io/kubernetes v1.32.0 h1:4BDBWSolqPrv8GC3YfZw0CJvh5kA1TPnoX0FxDVd+qc= k8s.io/kubernetes v1.32.0/go.mod h1:tiIKO63GcdPRBHW2WiUFm3C0eoLczl3f7qi56Dm1W8I= k8s.io/metrics v0.32.0 h1:70qJ3ZS/9DrtH0UA0NVBI6gW2ip2GAn9e7NtoKERpns= k8s.io/metrics v0.32.0/go.mod h1:skdg9pDjVjCPIQqmc5rBzDL4noY64ORhKu9KCPv1+QI= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go b/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go index 24e6269998b3..69b67862cdb0 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/certs.go @@ -17,34 +17,83 @@ limitations under the License. package main import ( - "io/ioutil" + "crypto/tls" + "os" + "path" + "sync" + "github.com/fsnotify/fsnotify" "k8s.io/klog/v2" ) -type certsContainer struct { - caCert, serverKey, serverCert []byte -} - type certsConfig struct { clientCaFile, tlsCertFile, tlsPrivateKey *string + reload *bool } func readFile(filePath string) []byte { - res, err := ioutil.ReadFile(filePath) + res, err := os.ReadFile(filePath) if err != nil { - klog.Errorf("Error reading certificate file at %s: %v", filePath, err) + klog.ErrorS(err, "Error reading certificate file", "file", filePath) return nil } - - klog.V(3).Infof("Successfully read %d bytes from %v", len(res), filePath) + klog.V(3).InfoS("Successfully read bytes from file", "bytes", len(res), "file", filePath) return res } -func initCerts(config certsConfig) certsContainer { - res := certsContainer{} - res.caCert = readFile(*config.clientCaFile) - res.serverCert = readFile(*config.tlsCertFile) - res.serverKey = readFile(*config.tlsPrivateKey) - return res +type certReloader struct { + tlsCertPath string + tlsKeyPath string + cert *tls.Certificate + mu sync.RWMutex +} + +func (cr *certReloader) start(stop <-chan struct{}) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + + if err = watcher.Add(path.Dir(cr.tlsCertPath)); err != nil { + return err + } + if err = watcher.Add(path.Dir(cr.tlsKeyPath)); err != nil { + return err + } + go func() { + defer watcher.Close() + for { + select { + case event := <-watcher.Events: + if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) { + klog.V(2).InfoS("New certificate found, reloading") + if err := cr.load(); err != nil { + klog.ErrorS(err, "Failed to reload certificate") + } + } + case err := <-watcher.Errors: + klog.Warningf("Error watching certificate files: %s", err) + case <-stop: + return + } + } + }() + return nil +} + +func (cr *certReloader) load() error { + cert, err := tls.LoadX509KeyPair(cr.tlsCertPath, cr.tlsKeyPath) + if err != nil { + return err + } + cr.mu.Lock() + defer cr.mu.Unlock() + cr.cert = &cert + return nil +} + +func (cr *certReloader) getCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + cr.mu.RLock() + defer cr.mu.RUnlock() + return cr.cert, nil } diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go new file mode 100644 index 000000000000..078de49fa23c --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/certs_test.go @@ -0,0 +1,150 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path" + "testing" + "time" +) + +func generateCerts(t *testing.T, org string, caCert *x509.Certificate, caKey *rsa.PrivateKey) ([]byte, []byte) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + Organization: []string{org}, + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + certKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Error(err) + } + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &certKey.PublicKey, caKey) + if err != nil { + t.Error(err) + } + + var certPem bytes.Buffer + err = pem.Encode(&certPem, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + t.Error(err) + } + + var certKeyPem bytes.Buffer + err = pem.Encode(&certKeyPem, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certKey), + }) + if err != nil { + t.Error(err) + } + return certPem.Bytes(), certKeyPem.Bytes() +} + +func TestKeypairReloader(t *testing.T) { + tempDir := t.TempDir() + caCert := &x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{ + Organization: []string{"ca"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(2, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Error(err) + } + caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caKey.PublicKey, caKey) + if err != nil { + t.Error(err) + } + caPath := path.Join(tempDir, "ca.crt") + caFile, err := os.Create(caPath) + if err != nil { + t.Error(err) + } + err = pem.Encode(caFile, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + t.Error(err) + } + + pub, privateKey := generateCerts(t, "first", caCert, caKey) + certPath := path.Join(tempDir, "cert.crt") + if err = os.WriteFile(certPath, pub, 0666); err != nil { + t.Error(err) + } + keyPath := path.Join(tempDir, "cert.key") + if err = os.WriteFile(keyPath, privateKey, 0666); err != nil { + t.Error(err) + } + + reloader := certReloader{ + tlsCertPath: certPath, + tlsKeyPath: keyPath, + } + stop := make(chan struct{}) + defer close(stop) + if err = reloader.start(stop); err != nil { + t.Error(err) + } + + pub, privateKey = generateCerts(t, "second", caCert, caKey) + if err = os.WriteFile(certPath, pub, 0666); err != nil { + t.Error(err) + } + if err = os.WriteFile(keyPath, privateKey, 0666); err != nil { + t.Error(err) + } + for { + tlsCert, err := reloader.getCertificate(nil) + if err != nil { + t.Error(err) + } + if tlsCert == nil { + continue + } + pubDER, _ := pem.Decode(pub) + if string(tlsCert.Certificate[0]) == string(pubDER.Bytes) { + return + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/config.go b/multidimensional-pod-autoscaler/pkg/admission-controller/config.go index babe54be8cd1..d1426567c76d 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/config.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/config.go @@ -19,6 +19,8 @@ package main import ( "context" "crypto/tls" + "fmt" + "strings" "time" admissionregistration "k8s.io/api/admissionregistration/v1" @@ -31,20 +33,65 @@ const ( webhookConfigName = "mpa-webhook-config" ) -func configTLS(serverCert, serverKey []byte) *tls.Config { - sCert, err := tls.X509KeyPair(serverCert, serverKey) - if err != nil { - klog.Fatal(err) +func configTLS(cfg certsConfig, minTlsVersion, ciphers string, stop <-chan struct{}) *tls.Config { + var tlsVersion uint16 + var ciphersuites []uint16 + reverseCipherMap := make(map[string]uint16) + + for _, c := range tls.CipherSuites() { + reverseCipherMap[c.Name] = c.ID + } + for _, c := range strings.Split(strings.ReplaceAll(ciphers, ",", ":"), ":") { + cipher, ok := reverseCipherMap[c] + if ok { + ciphersuites = append(ciphersuites, cipher) + } + } + if len(ciphersuites) == 0 { + ciphersuites = nil + } + + switch minTlsVersion { + case "": + fallthrough + case "tls1_2": + tlsVersion = tls.VersionTLS12 + case "tls1_3": + tlsVersion = tls.VersionTLS13 + default: + klog.Fatal(fmt.Errorf("Unable to determine value for --min-tls-version (%s), must be either tls1_2 or tls1_3", minTlsVersion)) + } + + config := &tls.Config{ + MinVersion: tlsVersion, + CipherSuites: ciphersuites, } - return &tls.Config{ - Certificates: []tls.Certificate{sCert}, + if *cfg.reload { + cr := certReloader{ + tlsCertPath: *cfg.tlsCertFile, + tlsKeyPath: *cfg.tlsPrivateKey, + } + if err := cr.load(); err != nil { + klog.Fatal(err) + } + if err := cr.start(stop); err != nil { + klog.Fatal(err) + } + config.GetCertificate = cr.getCertificate + } else { + cert, err := tls.LoadX509KeyPair(*cfg.tlsCertFile, *cfg.tlsPrivateKey) + if err != nil { + klog.Fatal(err) + } + config.Certificates = []tls.Certificate{cert} } + return config } // register this webhook admission controller with the kube-apiserver // by creating MutatingWebhookConfiguration. -func selfRegistration(clientset *kubernetes.Clientset, caCert []byte, namespace, serviceName, url string, registerByURL bool, timeoutSeconds int32) { - time.Sleep(10 * time.Second) +func selfRegistration(clientset kubernetes.Interface, caCert []byte, webHookDelay time.Duration, namespace, serviceName, url string, registerByURL bool, timeoutSeconds int32, selectedNamespace string, ignoredNamespaces []string, webHookFailurePolicy bool, webHookLabels string) { + time.Sleep(webHookDelay) client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations() _, err := client.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) if err == nil { @@ -62,11 +109,47 @@ func selfRegistration(clientset *kubernetes.Clientset, caCert []byte, namespace, RegisterClientConfig.URL = &url } sideEffects := admissionregistration.SideEffectClassNone - failurePolicy := admissionregistration.Ignore + + var failurePolicy admissionregistration.FailurePolicyType + if webHookFailurePolicy { + failurePolicy = admissionregistration.Fail + } else { + failurePolicy = admissionregistration.Ignore + } + RegisterClientConfig.CABundle = caCert + + var namespaceSelector metav1.LabelSelector + if len(ignoredNamespaces) > 0 { + namespaceSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: ignoredNamespaces, + }, + }, + } + } else if len(selectedNamespace) > 0 { + namespaceSelector = metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpIn, + Values: []string{selectedNamespace}, + }, + }, + } + } + webhookLabelsMap, err := convertLabelsToMap(webHookLabels) + if err != nil { + klog.ErrorS(err, "Unable to parse webhook labels") + webhookLabelsMap = map[string]string{} + } webhookConfig := &admissionregistration.MutatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: webhookConfigName, + Name: webhookConfigName, + Labels: webhookLabelsMap, }, Webhooks: []admissionregistration.MutatingWebhook{ { @@ -90,10 +173,11 @@ func selfRegistration(clientset *kubernetes.Clientset, caCert []byte, namespace, }, }, }, - FailurePolicy: &failurePolicy, - ClientConfig: RegisterClientConfig, - SideEffects: &sideEffects, - TimeoutSeconds: &timeoutSeconds, + FailurePolicy: &failurePolicy, + ClientConfig: RegisterClientConfig, + SideEffects: &sideEffects, + TimeoutSeconds: &timeoutSeconds, + NamespaceSelector: &namespaceSelector, }, }, } @@ -103,3 +187,29 @@ func selfRegistration(clientset *kubernetes.Clientset, caCert []byte, namespace, klog.V(3).Info("Self registration as MutatingWebhook succeeded.") } } + +// convertLabelsToMap convert the labels from string to map +// the valid labels format is "key1:value1,key2:value2", which could be converted to +// {"key1": "value1", "key2": "value2"} +func convertLabelsToMap(labels string) (map[string]string, error) { + m := make(map[string]string) + if labels == "" { + return m, nil + } + labels = strings.Trim(labels, "\"") + s := strings.Split(labels, ",") + for _, tag := range s { + kv := strings.SplitN(tag, ":", 2) + if len(kv) != 2 { + return map[string]string{}, fmt.Errorf("labels '%s' are invalid, the format should be: 'key1:value1,key2:value2'", labels) + } + key := strings.TrimSpace(kv[0]) + if key == "" { + return map[string]string{}, fmt.Errorf("labels '%s' are invalid, the format should be: 'key1:value1,key2:value2'", labels) + } + value := strings.TrimSpace(kv[1]) + m[key] = value + } + + return m, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go new file mode 100644 index 000000000000..69ed696519a8 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/config_test.go @@ -0,0 +1,373 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + admissionregistration "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestSelfRegistrationBase(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "key1:value1,key2:value2") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + assert.Equal(t, webhookConfigName, webhookConfig.Name, "expected webhook configuration name to match") + assert.Equal(t, webhookConfig.Labels, map[string]string{"key1": "value1", "key2": "value2"}, "expected webhook configuration labels to match") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + assert.Equal(t, "mpa.k8s.io", webhook.Name, "expected webhook name to match") + + PodRule := webhook.Rules[0] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create}, PodRule.Operations, "expected operations to match") + assert.Equal(t, []string{""}, PodRule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"v1"}, PodRule.APIVersions, "expected API versions to match") + assert.Equal(t, []string{"pods"}, PodRule.Resources, "expected resources to match") + + MPARule := webhook.Rules[1] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create, admissionregistration.Update}, MPARule.Operations, "expected operations to match") + assert.Equal(t, []string{"autoscaling.k8s.io"}, MPARule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"*"}, MPARule.APIVersions, "ehook.Rulxpected API versions to match") + assert.Equal(t, []string{"verticalpodautoscalers"}, MPARule.Resources, "expected resources to match") + + assert.Equal(t, admissionregistration.SideEffectClassNone, *webhook.SideEffects, "expected side effects to match") + assert.Equal(t, admissionregistration.Ignore, *webhook.FailurePolicy, "expected failure policy to match") + assert.Equal(t, caCert, webhook.ClientConfig.CABundle, "expected CA bundle to match") + assert.Equal(t, timeoutSeconds, *webhook.TimeoutSeconds, "expected timeout seconds to match") +} + +func TestSelfRegistrationWithURL(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.Nil(t, webhook.ClientConfig.Service, "expected service reference to be nil") + assert.NotNil(t, webhook.ClientConfig.URL, "expected URL to be set") + assert.Equal(t, url, *webhook.ClientConfig.URL, "expected URL to match") +} + +func TestSelfRegistrationWithOutURL(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.ClientConfig.Service, "expected service reference to be nil") + assert.Equal(t, webhook.ClientConfig.Service.Name, serviceName, "expected service name to be equal") + assert.Equal(t, webhook.ClientConfig.Service.Namespace, namespace, "expected service namespace to be equal") + + assert.Nil(t, webhook.ClientConfig.URL, "expected URL to be set") +} + +func TestSelfRegistrationWithIgnoredNamespaces(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{"test"} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.NamespaceSelector.MatchExpressions, "expected namespace selector not to be nil") + assert.Len(t, webhook.NamespaceSelector.MatchExpressions, 1, "expected one match expression") + + matchExpression := webhook.NamespaceSelector.MatchExpressions[0] + assert.Equal(t, matchExpression.Operator, metav1.LabelSelectorOpNotIn, "expected namespace operator to be OpNotIn") + assert.Equal(t, matchExpression.Values, ignoredNamespaces, "expected namespace selector match expression to be equal") +} + +func TestSelfRegistrationWithSelectedNamespaces(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, webhook.NamespaceSelector.MatchExpressions, "expected namespace selector not to be nil") + assert.Len(t, webhook.NamespaceSelector.MatchExpressions, 1, "expected one match expression") + + matchExpression := webhook.NamespaceSelector.MatchExpressions[0] + assert.Equal(t, metav1.LabelSelectorOpIn, matchExpression.Operator, "expected namespace operator to be OpIn") + assert.Equal(t, matchExpression.Operator, metav1.LabelSelectorOpIn, "expected namespace operator to be OpIn") + assert.Equal(t, matchExpression.Values, []string{selectedNamespace}, "expected namespace selector match expression to be equal") +} + +func TestSelfRegistrationWithFailurePolicy(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, true, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, *webhook.FailurePolicy, "expected failurePolicy not to be nil") + assert.Equal(t, *webhook.FailurePolicy, admissionregistration.Fail, "expected failurePolicy to be Fail") +} + +func TestSelfRegistrationWithOutFailurePolicy(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := false + timeoutSeconds := int32(32) + selectedNamespace := "test" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + + assert.NotNil(t, *webhook.FailurePolicy, "expected namespace selector not to be nil") + assert.Equal(t, *webhook.FailurePolicy, admissionregistration.Ignore, "expected failurePolicy to be Ignore") +} + +func TestSelfRegistrationWithInvalidLabels(t *testing.T) { + + testClientSet := fake.NewSimpleClientset() + caCert := []byte("fake") + webHookDelay := 0 * time.Second + namespace := "default" + serviceName := "mpa-service" + url := "http://example.com/" + registerByURL := true + timeoutSeconds := int32(32) + selectedNamespace := "" + ignoredNamespaces := []string{} + + selfRegistration(testClientSet, caCert, webHookDelay, namespace, serviceName, url, registerByURL, timeoutSeconds, selectedNamespace, ignoredNamespaces, false, "foo,bar") + + webhookConfigInterface := testClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations() + webhookConfig, err := webhookConfigInterface.Get(context.TODO(), webhookConfigName, metav1.GetOptions{}) + + assert.NoError(t, err, "expected no error fetching webhook configuration") + assert.Equal(t, webhookConfigName, webhookConfig.Name, "expected webhook configuration name to match") + assert.Equal(t, webhookConfig.Labels, map[string]string{}, "expected invalid webhook configuration labels to match") + + assert.Len(t, webhookConfig.Webhooks, 1, "expected one webhook configuration") + webhook := webhookConfig.Webhooks[0] + assert.Equal(t, "mpa.k8s.io", webhook.Name, "expected webhook name to match") + + PodRule := webhook.Rules[0] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create}, PodRule.Operations, "expected operations to match") + assert.Equal(t, []string{""}, PodRule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"v1"}, PodRule.APIVersions, "expected API versions to match") + assert.Equal(t, []string{"pods"}, PodRule.Resources, "expected resources to match") + + MPARule := webhook.Rules[1] + assert.Equal(t, []admissionregistration.OperationType{admissionregistration.Create, admissionregistration.Update}, MPARule.Operations, "expected operations to match") + assert.Equal(t, []string{"autoscaling.k8s.io"}, MPARule.APIGroups, "expected API groups to match") + assert.Equal(t, []string{"*"}, MPARule.APIVersions, "ehook.Rulxpected API versions to match") + assert.Equal(t, []string{"verticalpodautoscalers"}, MPARule.Resources, "expected resources to match") + + assert.Equal(t, admissionregistration.SideEffectClassNone, *webhook.SideEffects, "expected side effects to match") + assert.Equal(t, admissionregistration.Ignore, *webhook.FailurePolicy, "expected failure policy to match") + assert.Equal(t, caCert, webhook.ClientConfig.CABundle, "expected CA bundle to match") + assert.Equal(t, timeoutSeconds, *webhook.TimeoutSeconds, "expected timeout seconds to match") +} + +func TestConvertLabelsToMap(t *testing.T) { + testCases := []struct { + desc string + labels string + expectedOutput map[string]string + expectedError bool + }{ + { + desc: "should return empty map when tag is empty", + labels: "", + expectedOutput: map[string]string{}, + expectedError: false, + }, + { + desc: "single valid tag should be converted", + labels: "key:value", + expectedOutput: map[string]string{ + "key": "value", + }, + expectedError: false, + }, + { + desc: "multiple valid labels should be converted", + labels: "key1:value1,key2:value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "whitespaces should be trimmed", + labels: "key1:value1, key2:value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "whitespaces between keys and values should be trimmed", + labels: "key1 : value1,key2 : value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + desc: "should return error for invalid format", + labels: "foo,bar", + expectedOutput: nil, + expectedError: true, + }, + { + desc: "should return error for when key is missed", + labels: "key1:value1,:bar", + expectedOutput: nil, + expectedError: true, + }, + { + desc: "should strip additional quotes", + labels: "\"key1:value1,key2:value2\"", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + } + + for i, c := range testCases { + m, err := convertLabelsToMap(c.labels) + if c.expectedError { + assert.NotNil(t, err, "TestCase[%d]: %s", i, c.desc) + } else { + assert.Nil(t, err, "TestCase[%d]: %s", i, c.desc) + assert.Equal(t, m, c.expectedOutput, "expected labels map") + } + } +} diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go b/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go index 47711968abb3..17d767f4190c 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/logic/server.go @@ -81,7 +81,7 @@ func (s *AdmissionServer) admit(ctx context.Context, data []byte) (*admissionv1. handler, ok := s.resourceHandlers[admittedGroupResource] if ok { - patches, err = handler.GetPatches(ar.Request) + patches, err = handler.GetPatches(ctx, ar.Request) resource = handler.AdmissionResource() if handler.DisallowIncorrectObjects() && err != nil { diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/main.go b/multidimensional-pod-autoscaler/pkg/admission-controller/main.go index 40b90a360c33..9de357d2a41b 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/main.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/main.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "os" + "strings" "time" apiv1 "k8s.io/api/core/v1" @@ -33,9 +34,12 @@ import ( mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" metrics_admission "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" "k8s.io/client-go/informers" kube_client "k8s.io/client-go/kubernetes" @@ -44,8 +48,12 @@ import ( ) const ( - defaultResyncPeriod = 10 * time.Minute - statusUpdateInterval = 10 * time.Second + defaultResyncPeriod = 10 * time.Minute + statusUpdateInterval = 10 * time.Second + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + webHookDelay = 10 * time.Second ) var ( @@ -53,41 +61,52 @@ var ( clientCaFile: flag.String("client-ca-file", "/etc/tls-certs/caCert.pem", "Path to CA PEM file."), tlsCertFile: flag.String("tls-cert-file", "/etc/tls-certs/serverCert.pem", "Path to server certificate PEM file."), tlsPrivateKey: flag.String("tls-private-key", "/etc/tls-certs/serverKey.pem", "Path to server certificate key PEM file."), + reload: flag.Bool("reload-cert", false, "If set to true, reload leaf certificate."), } - port = flag.Int("port", 8000, "The port to listen on.") - address = flag.String("address", ":8944", "The address to expose Prometheus metrics.") - kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") - kubeApiQps = flag.Float64("kube-api-qps", 5.0, `QPS limit when making requests to Kubernetes apiserver`) - kubeApiBurst = flag.Float64("kube-api-burst", 10.0, `QPS burst limit when making requests to Kubernetes apiserver`) - namespace = os.Getenv("NAMESPACE") - serviceName = flag.String("webhook-service", "mpa-webhook", "Kubernetes service under which webhook is registered. Used when registerByURL is set to false.") - webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.") - webhookPort = flag.String("webhook-port", "", "Server Port for Webhook") - webhookTimeout = flag.Int("webhook-timeout-seconds", 30, "Timeout in seconds that the API server should wait for this webhook to respond before failing.") - registerWebhook = flag.Bool("register-webhook", true, "If set to true, admission webhook object will be created on start up to register with the API server.") - registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name") - mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + ciphers = flag.String("tls-ciphers", "", "A comma-separated or colon-separated list of ciphers to accept. Only works when min-tls-version is set to tls1_2.") + minTlsVersion = flag.String("min-tls-version", "tls1_2", "The minimum TLS version to accept. Must be set to either tls1_2 (default) or tls1_3.") + port = flag.Int("port", 8000, "The port to listen on.") + address = flag.String("address", ":8944", "The address to expose Prometheus metrics.") + // kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") + // kubeApiQps = flag.Float64("kube-api-qps", 5.0, `QPS limit when making requests to Kubernetes apiserver`) + // kubeApiBurst = flag.Float64("kube-api-burst", 10.0, `QPS burst limit when making requests to Kubernetes apiserver`) + namespace = os.Getenv("NAMESPACE") + serviceName = flag.String("webhook-service", "mpa-webhook", "Kubernetes service under which webhook is registered. Used when registerByURL is set to false.") + webhookAddress = flag.String("webhook-address", "", "Address under which webhook is registered. Used when registerByURL is set to true.") + webhookPort = flag.String("webhook-port", "", "Server Port for Webhook") + webhookTimeout = flag.Int("webhook-timeout-seconds", 30, "Timeout in seconds that the API server should wait for this webhook to respond before failing.") + webHookFailurePolicy = flag.Bool("webhook-failure-policy-fail", false, "If set to true, will configure the admission webhook failurePolicy to \"Fail\". Use with caution.") + registerWebhook = flag.Bool("register-webhook", true, "If set to true, admission webhook object will be created on start up to register with the API server.") + webhookLabels = flag.String("webhook-labels", "", "Comma separated list of labels to add to the webhook object. Format: key1:value1,key2:value2") + registerByURL = flag.Bool("register-by-url", false, "If set to true, admission webhook will be registered by URL (webhookAddress:webhookPort) instead of by service name") + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") ) func main() { + commonFlags := vpa_common.InitCommonFlags() klog.InitFlags(nil) + vpa_common.InitLoggingFlags() kube_flag.InitFlags() klog.V(1).Infof("Multi-dimensional Pod Autoscaler %s Admission Controller", common.MultidimPodAutoscalerVersion) + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } + healthCheck := metrics.NewHealthCheck(time.Minute) - metrics.Initialize(*address, healthCheck) metrics_admission.Register() + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) - certs := initCerts(*certsConfiguration) - klog.V(4).Infof("Certificates initialized!") - config := common.CreateKubeConfigOrDie(*kubeconfig, float32(*kubeApiQps), int(*kubeApiBurst)) + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) mpaClient := mpa_clientset.NewForConfigOrDie(config) mpaLister := mpa_api_util.NewMpasLister(mpaClient, make(chan struct{}), *mpaObjectNamespace) kubeClient := kube_client.NewForConfigOrDie(config) factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod) targetSelectorFetcher := target.NewMpaTargetSelectorFetcher(config, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) podPreprocessor := pod.NewDefaultPreProcessor() mpaPreprocessor := mpa.NewDefaultPreProcessor() var limitRangeCalculator limitrange.LimitRangeCalculator @@ -97,7 +116,7 @@ func main() { limitRangeCalculator = limitrange.NewNoopLimitsCalculator() } recommendationProvider := recommendation.NewProvider(limitRangeCalculator, mpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator)) - vpaMatcher := mpa.NewMatcher(mpaLister, targetSelectorFetcher) + mpaMatcher := mpa.NewMatcher(mpaLister, targetSelectorFetcher, controllerFetcher) hostname, err := os.Hostname() if err != nil { @@ -119,19 +138,20 @@ func main() { defer close(stopCh) calculators := []patch.Calculator{patch.NewResourceUpdatesCalculator(recommendationProvider), patch.NewObservedContainersCalculator()} - as := logic.NewAdmissionServer(podPreprocessor, mpaPreprocessor, limitRangeCalculator, vpaMatcher, calculators) + as := logic.NewAdmissionServer(podPreprocessor, mpaPreprocessor, limitRangeCalculator, mpaMatcher, calculators) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { as.Serve(w, r) healthCheck.UpdateLastActivity() }) server := &http.Server{ Addr: fmt.Sprintf(":%d", *port), - TLSConfig: configTLS(certs.serverCert, certs.serverKey), + TLSConfig: configTLS(*certsConfiguration, *minTlsVersion, *ciphers, stopCh), } url := fmt.Sprintf("%v:%v", *webhookAddress, *webhookPort) + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") go func() { if *registerWebhook { - selfRegistration(kubeClient, certs.caCert, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout)) + selfRegistration(kubeClient, readFile(*certsConfiguration.clientCaFile), webHookDelay, namespace, *serviceName, url, *registerByURL, int32(*webhookTimeout), commonFlags.VpaObjectNamespace, ignoredNamespaces, *webHookFailurePolicy, *webhookLabels) } // Start status updates after the webhook is initialized. statusUpdater.Run(stopCh) diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go index 4b9759c5eee8..675178dbe7c2 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/handler.go @@ -17,6 +17,7 @@ limitations under the License. package mpa import ( + "context" "encoding/json" "fmt" @@ -72,7 +73,7 @@ func (h *resourceHandler) DisallowIncorrectObjects() bool { } // GetPatches builds patches for VPA in given admission request. -func (h *resourceHandler) GetPatches(ar *v1.AdmissionRequest) ([]resource.PatchRecord, error) { +func (h *resourceHandler) GetPatches(_ context.Context, ar *v1.AdmissionRequest) ([]resource.PatchRecord, error) { raw, isCreate := ar.Object.Raw, ar.Operation == v1.Create mpa, err := parseMPA(raw) if err != nil { diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go index 1d105834f286..46d05b7e4efb 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher.go @@ -17,6 +17,8 @@ limitations under the License. package mpa import ( + "context" + core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" @@ -24,52 +26,73 @@ import ( "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" "k8s.io/klog/v2" ) // Matcher is capable of returning a single matching MPA object // for a pod. Will return nil if no matching object is found. type Matcher interface { - GetMatchingMPA(pod *core.Pod) *mpa_types.MultidimPodAutoscaler + GetMatchingMPA(ctx context.Context, pod *core.Pod) *mpa_types.MultidimPodAutoscaler } type matcher struct { - mpaLister mpa_lister.MultidimPodAutoscalerLister - selectorFetcher target.MpaTargetSelectorFetcher + mpaLister mpa_lister.MultidimPodAutoscalerLister + selectorFetcher target.MpaTargetSelectorFetcher + controllerFetcher controllerfetcher.ControllerFetcher } // NewMatcher returns a new MPA matcher. func NewMatcher(mpaLister mpa_lister.MultidimPodAutoscalerLister, - selectorFetcher target.MpaTargetSelectorFetcher) Matcher { + selectorFetcher target.MpaTargetSelectorFetcher, + controllerFetcher controllerfetcher.ControllerFetcher) Matcher { return &matcher{mpaLister: mpaLister, - selectorFetcher: selectorFetcher} + selectorFetcher: selectorFetcher, + controllerFetcher: controllerFetcher} } -func (m *matcher) GetMatchingMPA(pod *core.Pod) *mpa_types.MultidimPodAutoscaler { +func (m *matcher) GetMatchingMPA(ctx context.Context, pod *core.Pod) *mpa_types.MultidimPodAutoscaler { + parentController, err := mpa_api_util.FindParentControllerForPod(ctx, pod, m.controllerFetcher) + if err != nil { + klog.ErrorS(err, "Failed to get parent controller for pod", "pod", klog.KObj(pod)) + return nil + } + if parentController == nil { + return nil + } + configs, err := m.mpaLister.MultidimPodAutoscalers(pod.Namespace).List(labels.Everything()) if err != nil { klog.Errorf("failed to get mpa configs: %v", err) return nil } - onConfigs := make([]*mpa_api_util.MpaWithSelector, 0) + + var controllingMpa *mpa_types.MultidimPodAutoscaler for _, mpaConfig := range configs { if mpa_api_util.GetUpdateMode(mpaConfig) == vpa_types.UpdateModeOff { continue } - selector, err := m.selectorFetcher.Fetch(mpaConfig) + if mpaConfig.Spec.ScaleTargetRef == nil { + klog.V(5).InfoS("Skipping MPA object because scaleTargetRef is not defined.", "mpa", klog.KObj(mpaConfig)) + continue + } + if mpaConfig.Spec.ScaleTargetRef.Kind != parentController.Kind || + mpaConfig.Namespace != parentController.Namespace || + mpaConfig.Spec.ScaleTargetRef.Name != parentController.Name { + continue // This pod is not associated to the right controller + } + + selector, err := m.selectorFetcher.Fetch(ctx, mpaConfig) if err != nil { - klog.V(3).Infof("skipping MPA object %v because we cannot fetch selector: %s", mpaConfig.Name, err) + klog.V(3).InfoS("Skipping MPA object because we cannot fetch selector", "mpa", klog.KObj(mpaConfig), "error", err) continue } - onConfigs = append(onConfigs, &mpa_api_util.MpaWithSelector{ - Mpa: mpaConfig, - Selector: selector, - }) - } - klog.V(2).Infof("Let's choose from %d configs for pod %s/%s", len(onConfigs), pod.Namespace, pod.Name) - result := mpa_api_util.GetControllingMPAForPod(pod, onConfigs) - if result != nil { - return result.Mpa + + mpaWithSelector := &mpa_api_util.MpaWithSelector{Mpa: mpaConfig, Selector: selector} + if mpa_api_util.PodMatchesMPA(pod, mpaWithSelector) && mpa_api_util.Stronger(mpaConfig, controllingMpa) { + controllingMpa = mpaConfig + } } - return nil + + return controllingMpa } diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go index 0d97d1c729ff..cba79b2d730a 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/mpa/matcher_test.go @@ -17,6 +17,7 @@ limitations under the License. package mpa import ( + "context" "testing" core "k8s.io/api/core/v1" @@ -26,6 +27,7 @@ import ( target_mock "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target/mock" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/test" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" test_vpa "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" "github.com/golang/mock/gomock" @@ -115,10 +117,16 @@ func TestGetMatchingVpa(t *testing.T) { mpaLister := &test.MultidimPodAutoscalerListerMock{} mpaLister.On("MultidimPodAutoscalers", "default").Return(mpaNamespaceLister) - mockSelectorFetcher.EXPECT().Fetch(gomock.Any()).AnyTimes().Return(parseLabelSelector(tc.labelSelector), nil) - matcher := NewMatcher(mpaLister, mockSelectorFetcher) + if tc.labelSelector != "" { + mockSelectorFetcher.EXPECT().Fetch(gomock.Any()).AnyTimes().Return(parseLabelSelector(tc.labelSelector), nil) + } + // This test is using a FakeControllerFetcher which returns the same ownerRef that is passed to it. + // In other words, it cannot go through the hierarchy of controllers like "ReplicaSet => Deployment" + // For this reason we are using "StatefulSet" as the ownerRef kind in the test, since it is a direct link. + // The hierarchy part is being test in the "TestControllerFetcher" test. + matcher := NewMatcher(mpaLister, mockSelectorFetcher, controllerfetcher.FakeControllerFetcher{}) - mpa := matcher.GetMatchingMPA(tc.pod) + mpa := matcher.GetMatchingMPA(context.Background(), tc.pod) if tc.expectedFound && assert.NotNil(t, mpa) { assert.Equal(t, tc.expectedVpaName, mpa.Name) } else { diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go index 61e89037b23e..fa076feae65f 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler.go @@ -17,6 +17,7 @@ limitations under the License. package pod import ( + "context" "encoding/json" "fmt" @@ -63,7 +64,7 @@ func (h *resourceHandler) DisallowIncorrectObjects() bool { } // GetPatches builds patches for Pod in given admission request. -func (h *resourceHandler) GetPatches(ar *admissionv1.AdmissionRequest) ([]resource_admission.PatchRecord, error) { +func (h *resourceHandler) GetPatches(ctx context.Context, ar *admissionv1.AdmissionRequest) ([]resource_admission.PatchRecord, error) { if ar.Resource.Version != "v1" { return nil, fmt.Errorf("only v1 Pods are supported") } @@ -77,7 +78,7 @@ func (h *resourceHandler) GetPatches(ar *admissionv1.AdmissionRequest) ([]resour pod.Namespace = namespace } klog.V(4).Infof("Admitting pod %v", pod.ObjectMeta) - controllingMpa := h.mpaMatcher.GetMatchingMPA(&pod) + controllingMpa := h.mpaMatcher.GetMatchingMPA(ctx, &pod) if controllingMpa == nil { klog.V(4).Infof("No matching MPA found for pod %s/%s", pod.Namespace, pod.Name) return []resource_admission.PatchRecord{}, nil diff --git a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go index 2b7eebb795e1..868e542e91ad 100644 --- a/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go +++ b/multidimensional-pod-autoscaler/pkg/admission-controller/resource/pod/handler_test.go @@ -17,6 +17,7 @@ limitations under the License. package pod import ( + "context" "fmt" "testing" @@ -43,7 +44,7 @@ type fakeMpaMatcher struct { mpa *mpa_types.MultidimPodAutoscaler } -func (m *fakeMpaMatcher) GetMatchingMPA(_ *apiv1.Pod) *mpa_types.MultidimPodAutoscaler { +func (m *fakeMpaMatcher) GetMatchingMPA(_ context.Context, _ *apiv1.Pod) *mpa_types.MultidimPodAutoscaler { return m.mpa } @@ -176,7 +177,7 @@ func TestGetPatches(t *testing.T) { fppp := &fakePodPreProcessor{tc.podPreProcessorError} fvm := &fakeMpaMatcher{mpa: tc.mpa} h := NewResourceHandler(fppp, fvm, tc.calculators) - patches, err := h.GetPatches(&admissionv1.AdmissionRequest{ + patches, err := h.GetPatches(context.Background(), &admissionv1.AdmissionRequest{ Resource: v1.GroupVersionResource{ Version: "v1", }, diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index 1f08c0c2f9f9..faf8dd3b55b4 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -31,11 +31,11 @@ import ( mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input/metrics" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" - "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/metrics" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/oom" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/spec" vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" @@ -73,7 +73,7 @@ type ClusterStateFeeder interface { InitFromCheckpoints() // LoadMPAs updates clusterState with current state of MPAs. - LoadMPAs() + LoadMPAs(ctx context.Context) // LoadPods updates clusterState with current specification of Pods and their Containers. LoadPods() @@ -101,6 +101,7 @@ type ClusterStateFeederFactory struct { MemorySaveMode bool ControllerFetcher controllerfetcher.ControllerFetcher RecommenderName string + IgnoredNamespaces []string } // Make creates new ClusterStateFeeder with internal data providers, based on kube client. @@ -118,6 +119,7 @@ func (m ClusterStateFeederFactory) Make() *clusterStateFeeder { memorySaveMode: m.MemorySaveMode, controllerFetcher: m.ControllerFetcher, recommenderName: m.RecommenderName, + ignoredNamespaces: m.IgnoredNamespaces, } } @@ -235,6 +237,7 @@ type clusterStateFeeder struct { memorySaveMode bool controllerFetcher controllerfetcher.ControllerFetcher recommenderName string + ignoredNamespaces []string PodLister v1lister.PodLister // For HPA. } @@ -287,7 +290,7 @@ func (feeder *clusterStateFeeder) setMpaCheckpoint(checkpoint *mpa_types.Multidi func (feeder *clusterStateFeeder) InitFromCheckpoints() { klog.V(3).Info("Initializing MPA from checkpoints") - feeder.LoadMPAs() + feeder.LoadMPAs(context.TODO()) namespaces := make(map[string]bool) for _, v := range feeder.clusterState.Mpas { @@ -318,7 +321,7 @@ func (feeder *clusterStateFeeder) GetPodLister() v1lister.PodLister { func (feeder *clusterStateFeeder) GarbageCollectCheckpoints() { klog.V(3).Info("Starting garbage collection of checkpoints") - feeder.LoadMPAs() + feeder.LoadMPAs(context.TODO()) namspaceList, err := feeder.coreClient.Namespaces().List(context.TODO(), metav1.ListOptions{}) if err != nil { @@ -386,7 +389,7 @@ func filterMPAs(feeder *clusterStateFeeder, allMpaCRDs []*mpa_types.MultidimPodA } // Fetch MPA objects and load them into the cluster state. -func (feeder *clusterStateFeeder) LoadMPAs() { +func (feeder *clusterStateFeeder) LoadMPAs(ctx context.Context) { // List MPA API objects. allMpaCRDs, err := feeder.mpaLister.List(labels.Everything()) if err != nil { @@ -406,7 +409,7 @@ func (feeder *clusterStateFeeder) LoadMPAs() { MpaName: mpaCRD.Name, } - selector, conditions := feeder.getSelector(mpaCRD) + selector, conditions := feeder.getSelector(ctx, mpaCRD) klog.V(4).Infof("Using selector %s for MPA %s/%s", selector.String(), mpaCRD.Namespace, mpaCRD.Name) if feeder.clusterState.AddOrUpdateMpa(mpaCRD, selector) == nil { @@ -535,7 +538,7 @@ type condition struct { message string } -func (feeder *clusterStateFeeder) validateTargetRef(mpa *mpa_types.MultidimPodAutoscaler) (bool, condition) { +func (feeder *clusterStateFeeder) validateTargetRef(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (bool, condition) { if mpa.Spec.ScaleTargetRef == nil { return false, condition{} } @@ -547,7 +550,7 @@ func (feeder *clusterStateFeeder) validateTargetRef(mpa *mpa_types.MultidimPodAu }, ApiVersion: mpa.Spec.ScaleTargetRef.APIVersion, } - top, err := feeder.controllerFetcher.FindTopMostWellKnownOrScalable(&k) + top, err := feeder.controllerFetcher.FindTopMostWellKnownOrScalable(ctx, &k) if err != nil { return false, condition{conditionType: mpa_types.ConfigUnsupported, delete: false, message: fmt.Sprintf("Error checking if target is a topmost well-known or scalable controller: %s", err)} } @@ -560,10 +563,10 @@ func (feeder *clusterStateFeeder) validateTargetRef(mpa *mpa_types.MultidimPodAu return true, condition{} } -func (feeder *clusterStateFeeder) getSelector(mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, []condition) { - selector, fetchErr := feeder.selectorFetcher.Fetch(mpa) +func (feeder *clusterStateFeeder) getSelector(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, []condition) { + selector, fetchErr := feeder.selectorFetcher.Fetch(ctx, mpa) if selector != nil { - validTargetRef, unsupportedCondition := feeder.validateTargetRef(mpa) + validTargetRef, unsupportedCondition := feeder.validateTargetRef(ctx, mpa) if !validTargetRef { return labels.Nothing(), []condition{ unsupportedCondition, diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index 9288dac0f483..41b348eb472b 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -17,6 +17,7 @@ limitations under the License. package input import ( + "context" "fmt" "testing" "time" @@ -48,7 +49,7 @@ type fakeControllerFetcher struct { scaleNamespacer scale.ScalesGetter } -func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(_ *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { +func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(_ context.Context, _ *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { return f.key, f.err } @@ -329,7 +330,7 @@ func TestLoadPods(t *testing.T) { if tc.expectedMpaFetch { targetSelectorFetcher.EXPECT().Fetch(mpa).Return(tc.selector, tc.fetchSelectorError) } - clusterStateFeeder.LoadMPAs() + clusterStateFeeder.LoadMPAs(context.Background()) mpaID := model.MpaID{ Namespace: mpa.Namespace, diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go new file mode 100644 index 000000000000..6e0de6a2039a --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client.go @@ -0,0 +1,118 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + recommender_metrics "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/klog/v2" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" +) + +// ContainerMetricsSnapshot contains information about usage of certain container within defined time window. +type ContainerMetricsSnapshot struct { + // ID identifies a specific container those metrics are coming from. + ID model.ContainerID + // End time of the measurement interval. + SnapshotTime time.Time + // Duration of the measurement interval, which is [SnapshotTime - SnapshotWindow, SnapshotTime]. + SnapshotWindow time.Duration + // Actual usage of the resources over the measurement interval. + Usage model.Resources +} + +// MetricsClient provides simple metrics on resources usage on container level. +type MetricsClient interface { + // GetContainersMetrics returns an array of ContainerMetricsSnapshots, + // representing resource usage for every running container in the cluster + GetContainersMetrics() ([]*ContainerMetricsSnapshot, error) +} + +type metricsClient struct { + source PodMetricsLister + namespace string + clientName string +} + +// NewMetricsClient creates new instance of MetricsClient, which is used by recommender. +// namespace limits queries to particular namespace, use k8sapiv1.NamespaceAll to select all namespaces. +func NewMetricsClient(source PodMetricsLister, namespace, clientName string) MetricsClient { + return &metricsClient{ + source: source, + namespace: namespace, + clientName: clientName, + } +} + +func (c *metricsClient) GetContainersMetrics() ([]*ContainerMetricsSnapshot, error) { + var metricsSnapshots []*ContainerMetricsSnapshot + + podMetricsList, err := c.source.List(context.TODO(), c.namespace, metav1.ListOptions{}) + recommender_metrics.RecordMetricsServerResponse(err, c.clientName) + if err != nil { + return nil, err + } + klog.V(3).InfoS("podMetrics retrieved for all namespaces", "podMetrics", len(podMetricsList.Items)) + for _, podMetrics := range podMetricsList.Items { + metricsSnapshotsForPod := createContainerMetricsSnapshots(podMetrics) + metricsSnapshots = append(metricsSnapshots, metricsSnapshotsForPod...) + } + return metricsSnapshots, nil +} + +func createContainerMetricsSnapshots(podMetrics v1beta1.PodMetrics) []*ContainerMetricsSnapshot { + snapshots := make([]*ContainerMetricsSnapshot, len(podMetrics.Containers)) + for i, containerMetrics := range podMetrics.Containers { + snapshots[i] = newContainerMetricsSnapshot(containerMetrics, podMetrics) + } + return snapshots +} + +func newContainerMetricsSnapshot(containerMetrics v1beta1.ContainerMetrics, podMetrics v1beta1.PodMetrics) *ContainerMetricsSnapshot { + usage := calculateUsage(containerMetrics.Usage) + + return &ContainerMetricsSnapshot{ + ID: model.ContainerID{ + ContainerName: containerMetrics.Name, + PodID: model.PodID{ + Namespace: podMetrics.Namespace, + PodName: podMetrics.Name, + }, + }, + Usage: usage, + SnapshotTime: podMetrics.Timestamp.Time, + SnapshotWindow: podMetrics.Window.Duration, + } +} + +func calculateUsage(containerUsage k8sapiv1.ResourceList) model.Resources { + cpuQuantity := containerUsage[k8sapiv1.ResourceCPU] + cpuMillicores := cpuQuantity.MilliValue() + + memoryQuantity := containerUsage[k8sapiv1.ResourceMemory] + memoryBytes := memoryQuantity.Value() + + return model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuMillicores), + model.ResourceMemory: model.ResourceAmount(memoryBytes), + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go new file mode 100644 index 000000000000..3e29284fadb6 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetContainersMetricsReturnsEmptyList(t *testing.T) { + tc := newEmptyMetricsClientTestCase() + emptyMetricsClient := tc.createFakeMetricsClient() + + containerMetricsSnapshots, err := emptyMetricsClient.GetContainersMetrics() + + assert.NoError(t, err) + assert.Empty(t, containerMetricsSnapshots, "should be empty for empty MetricsGetter") +} + +func TestGetContainersMetricsReturnsResults(t *testing.T) { + tc := newMetricsClientTestCase() + fakeMetricsClient := tc.createFakeMetricsClient() + + snapshots, err := fakeMetricsClient.GetContainersMetrics() + + assert.NoError(t, err) + assert.Len(t, snapshots, len(tc.getAllSnaps()), "It should return right number of snapshots") + for _, snap := range snapshots { + assert.Contains(t, tc.getAllSnaps(), snap, "One of returned ContainerMetricsSnapshot is different then expected ") + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go new file mode 100644 index 000000000000..5f967ae741aa --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_client_test_util.go @@ -0,0 +1,138 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "math/big" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/runtime" + core "k8s.io/client-go/testing" + + metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1" + "k8s.io/metrics/pkg/client/clientset/versioned/fake" + + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" +) + +type metricsClientTestCase struct { + snapshotTimestamp time.Time + snapshotWindow time.Duration + namespace *v1.Namespace + pod1Snaps, pod2Snaps []*ContainerMetricsSnapshot +} + +func newMetricsClientTestCase() *metricsClientTestCase { + namespaceName := "test-namespace" + + testCase := &metricsClientTestCase{ + snapshotTimestamp: time.Now(), + snapshotWindow: time.Duration(1234), + namespace: &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}}, + } + + id1 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod1"}, ContainerName: "Name1"} + id2 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod1"}, ContainerName: "Name2"} + id3 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod2"}, ContainerName: "Name1"} + id4 := model.ContainerID{PodID: model.PodID{Namespace: namespaceName, PodName: "Pod2"}, ContainerName: "Name2"} + + testCase.pod1Snaps = append(testCase.pod1Snaps, testCase.newContainerMetricsSnapshot(id1, 400, 333)) + testCase.pod1Snaps = append(testCase.pod1Snaps, testCase.newContainerMetricsSnapshot(id2, 800, 666)) + testCase.pod2Snaps = append(testCase.pod2Snaps, testCase.newContainerMetricsSnapshot(id3, 401, 334)) + testCase.pod2Snaps = append(testCase.pod2Snaps, testCase.newContainerMetricsSnapshot(id4, 801, 667)) + + return testCase +} + +func newEmptyMetricsClientTestCase() *metricsClientTestCase { + return &metricsClientTestCase{} +} + +func (tc *metricsClientTestCase) newContainerMetricsSnapshot(id model.ContainerID, cpuUsage int64, memUsage int64) *ContainerMetricsSnapshot { + return &ContainerMetricsSnapshot{ + ID: id, + SnapshotTime: tc.snapshotTimestamp, + SnapshotWindow: tc.snapshotWindow, + Usage: model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuUsage), + model.ResourceMemory: model.ResourceAmount(memUsage), + }, + } +} + +func (tc *metricsClientTestCase) createFakeMetricsClient() MetricsClient { + fakeMetricsGetter := &fake.Clientset{} + fakeMetricsGetter.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) { + return true, tc.getFakePodMetricsList(), nil + }) + return NewMetricsClient(NewPodMetricsesSource(fakeMetricsGetter.MetricsV1beta1()), "", "fake") +} + +func (tc *metricsClientTestCase) getFakePodMetricsList() *metricsapi.PodMetricsList { + metrics := &metricsapi.PodMetricsList{} + if tc.pod1Snaps != nil && tc.pod2Snaps != nil { + metrics.Items = append(metrics.Items, makePodMetrics(tc.pod1Snaps)) + metrics.Items = append(metrics.Items, makePodMetrics(tc.pod2Snaps)) + } + return metrics +} + +func makePodMetrics(snaps []*ContainerMetricsSnapshot) metricsapi.PodMetrics { + firstSnap := snaps[0] + podMetrics := metricsapi.PodMetrics{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: firstSnap.ID.Namespace, + Name: firstSnap.ID.PodName, + }, + Timestamp: metav1.Time{Time: firstSnap.SnapshotTime}, + Window: metav1.Duration{Duration: firstSnap.SnapshotWindow}, + Containers: make([]metricsapi.ContainerMetrics, len(snaps)), + } + + for i, snap := range snaps { + resourceList := calculateResourceList(snap.Usage) + podMetrics.Containers[i] = metricsapi.ContainerMetrics{ + Name: snap.ID.ContainerName, + Usage: resourceList, + } + } + return podMetrics +} + +func calculateResourceList(usage model.Resources) k8sapiv1.ResourceList { + cpuCores := big.NewRat(int64(usage[model.ResourceCPU]), 1000) + cpuQuantityString := cpuCores.FloatString(3) + + memoryBytes := big.NewInt(int64(usage[model.ResourceMemory])) + memoryQuantityString := memoryBytes.String() + + resourceMap := map[k8sapiv1.ResourceName]resource.Quantity{ + k8sapiv1.ResourceCPU: resource.MustParse(cpuQuantityString), + k8sapiv1.ResourceMemory: resource.MustParse(memoryQuantityString), + } + return k8sapiv1.ResourceList(resourceMap) +} + +func (tc *metricsClientTestCase) getAllSnaps() []*ContainerMetricsSnapshot { + return append(tc.pod1Snaps, tc.pod2Snaps...) +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go new file mode 100644 index 000000000000..1df8bf76a2fe --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "time" + + k8sapiv1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" + resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" + "k8s.io/metrics/pkg/client/external_metrics" +) + +// PodMetricsLister wraps both metrics-client and External Metrics +type PodMetricsLister interface { + List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) +} + +// podMetricsSource is the metrics-client source of metrics. +type podMetricsSource struct { + metricsGetter resourceclient.PodMetricsesGetter +} + +// NewPodMetricsesSource Returns a Source-wrapper around PodMetricsesGetter. +func NewPodMetricsesSource(source resourceclient.PodMetricsesGetter) PodMetricsLister { + return podMetricsSource{metricsGetter: source} +} + +func (s podMetricsSource) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { + podMetricsInterface := s.metricsGetter.PodMetricses(namespace) + return podMetricsInterface.List(ctx, opts) +} + +// externalMetricsClient is the External Metrics source of metrics. +type externalMetricsClient struct { + externalClient external_metrics.ExternalMetricsClient + options ExternalClientOptions + clusterState *model.ClusterState +} + +// ExternalClientOptions specifies parameters for using an External Metrics Client. +type ExternalClientOptions struct { + ResourceMetrics map[k8sapiv1.ResourceName]string + // Label to use for the container name. + ContainerNameLabel string +} + +// NewExternalClient returns a Source for an External Metrics Client. +func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options ExternalClientOptions) PodMetricsLister { + extClient, err := external_metrics.NewForConfig(c) + if err != nil { + klog.Fatalf("Failed initializing external metrics client: %v", err) + } + return &externalMetricsClient{ + externalClient: extClient, + options: options, + clusterState: clusterState, + } +} + +func (s *externalMetricsClient) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { + result := v1beta1.PodMetricsList{} + + for _, mpa := range s.clusterState.Mpas { + if mpa.PodCount == 0 { + continue + } + + if namespace != "" && mpa.ID.Namespace != namespace { + continue + } + + nsClient := s.externalClient.NamespacedMetrics(mpa.ID.Namespace) + pods := s.clusterState.GetMatchingPods(mpa) + + for _, pod := range pods { + podNameReq, err := labels.NewRequirement("pod", selection.Equals, []string{pod.PodName}) + if err != nil { + return nil, err + } + selector := mpa.PodSelector.Add(*podNameReq) + podMets := v1beta1.PodMetrics{ + TypeMeta: v1.TypeMeta{}, + ObjectMeta: v1.ObjectMeta{Namespace: mpa.ID.Namespace, Name: pod.PodName}, + Window: v1.Duration{}, + Containers: make([]v1beta1.ContainerMetrics, 0), + } + // Query each resource in turn, then assemble back to a single []ContainerMetrics. + containerMetrics := make(map[string]k8sapiv1.ResourceList) + for resourceName, metricName := range s.options.ResourceMetrics { + m, err := nsClient.List(metricName, selector) + if err != nil { + return nil, err + } + if m == nil || len(m.Items) == 0 { + klog.V(4).InfoS("External Metrics Query for MPA: No items", "mpa", klog.KRef(mpa.ID.Namespace, mpa.ID.MpaName), "resource", resourceName, "metric", metricName) + continue + } + klog.V(4).InfoS("External Metrics Query for MPA", "mpa", klog.KRef(mpa.ID.Namespace, mpa.ID.MpaName), "resource", resourceName, "metric", metricName, "itemCount", len(m.Items), "firstItem", m.Items[0]) + podMets.Timestamp = m.Items[0].Timestamp + if m.Items[0].WindowSeconds != nil { + podMets.Window = v1.Duration{Duration: time.Duration(*m.Items[0].WindowSeconds) * time.Second} + } + for _, val := range m.Items { + ctrName, hasCtrName := val.MetricLabels[s.options.ContainerNameLabel] + if !hasCtrName { + continue + } + if containerMetrics[ctrName] == nil { + containerMetrics[ctrName] = make(k8sapiv1.ResourceList) + } + containerMetrics[ctrName][resourceName] = val.Value + } + + } + for cname, res := range containerMetrics { + podMets.Containers = append(podMets.Containers, v1beta1.ContainerMetrics{Name: cname, Usage: res}) + } + result.Items = append(result.Items, podMets) + + } + } + return &result, nil +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/main.go b/multidimensional-pod-autoscaler/pkg/recommender/main.go index 501975e4b61b..32d631849c3c 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/main.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/main.go @@ -19,46 +19,67 @@ package main import ( "context" "flag" + "os" + "strings" "time" - "k8s.io/apimachinery/pkg/util/wait" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/checkpoint" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/model" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/routines" + "k8s.io/metrics/pkg/client/custom_metrics" + "k8s.io/metrics/pkg/client/external_metrics" apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/autoscaler/multidimensional-pod-autoscaler/common" + mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" - "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/routines" + input_metrics "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/recommender/input/metrics" + "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" metrics_recommender "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/metrics/recommender" + mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/logic" vpa_model "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" metrics_quality "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/quality" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" + "k8s.io/client-go/discovery" + cacheddiscovery "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/informers" kube_client "k8s.io/client-go/kubernetes" "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" kube_flag "k8s.io/component-base/cli/flag" + componentbaseconfig "k8s.io/component-base/config" + componentbaseoptions "k8s.io/component-base/config/options" klog "k8s.io/klog/v2" - - "k8s.io/client-go/discovery" - cacheddiscovery "k8s.io/client-go/discovery/cached" hpa_metrics "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" - "k8s.io/metrics/pkg/client/custom_metrics" - "k8s.io/metrics/pkg/client/external_metrics" ) var ( - recommenderName = flag.String("recommender-name", input.DefaultRecommenderName, "Set the recommender name. Recommender will generate recommendations for MPAs that configure the same recommender name. If the recommender name is left as default it will also generate recommendations that don't explicitly specify recommender. You shouldn't run two recommenders with the same name in a cluster.") - metricsFetcherInterval = flag.Duration("recommender-interval", 1*time.Minute, `How often metrics should be fetched`) - checkpointsGCInterval = flag.Duration("checkpoints-gc-interval", 10*time.Minute, `How often orphaned checkpoints should be garbage collected`) - prometheusAddress = flag.String("prometheus-address", "", `Where to reach for Prometheus metrics`) - prometheusJobName = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`) - address = flag.String("address", ":8942", "The address to expose Prometheus metrics.") - kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") - kubeApiQps = flag.Float64("kube-api-qps", 5.0, `QPS limit when making requests to Kubernetes apiserver`) - kubeApiBurst = flag.Float64("kube-api-burst", 10.0, `QPS burst limit when making requests to Kubernetes apiserver`) - - storage = flag.String("storage", "", `Specifies storage mode. Supported values: prometheus, checkpoint (default)`) - // prometheus history provider configs + recommenderName = flag.String("recommender-name", input.DefaultRecommenderName, "Set the recommender name. Recommender will generate recommendations for MPAs that configure the same recommender name. If the recommender name is left as default it will also generate recommendations that don't explicitly specify recommender. You shouldn't run two recommenders with the same name in a cluster.") + metricsFetcherInterval = flag.Duration("recommender-interval", 1*time.Minute, `How often metrics should be fetched`) + checkpointsGCInterval = flag.Duration("checkpoints-gc-interval", 10*time.Minute, `How often orphaned checkpoints should be garbage collected`) + address = flag.String("address", ":8942", "The address to expose Prometheus metrics.") + storage = flag.String("storage", "", `Specifies storage mode. Supported values: prometheus, checkpoint (default)`) + memorySaver = flag.Bool("memory-saver", false, `If true, only track pods which have an associated MPA`) + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects and pod stats. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") +) + +// prometheus history provider configs +var ( + prometheusAddress = flag.String("prometheus-address", "", `Where to reach for Prometheus metrics`) + prometheusJobName = flag.String("prometheus-cadvisor-job-name", "kubernetes-cadvisor", `Name of the prometheus job name which scrapes the cAdvisor metrics`) historyLength = flag.String("history-length", "8d", `How much time back prometheus have to be queried to get historical metrics`) historyResolution = flag.String("history-resolution", "1h", `Resolution at which Prometheus is queried for historical metrics`) queryTimeout = flag.String("prometheus-query-timeout", "5m", `How long to wait before killing long queries`) @@ -69,7 +90,15 @@ var ( ctrNamespaceLabel = flag.String("container-namespace-label", "namespace", `Label name to look for container namespaces`) ctrPodNameLabel = flag.String("container-pod-name-label", "pod_name", `Label name to look for container pod names`) ctrNameLabel = flag.String("container-name-label", "name", `Label name to look for container names`) - mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects and pod stats. Empty means all namespaces will be used.") + username = flag.String("username", "", "The username used in the prometheus server basic auth") + password = flag.String("password", "", "The password used in the prometheus server basic auth") +) + +// External metrics provider flags +var ( + useExternalMetrics = flag.Bool("use-external-metrics", false, "ALPHA. Use an external metrics provider instead of metrics_server.") + externalCpuMetric = flag.String("external-metrics-cpu-metric", "", "ALPHA. Metric to use with external metrics provider for CPU usage.") + externalMemoryMetric = flag.String("external-metrics-memory-metric", "", "ALPHA. Metric to use with external metrics provider for memory usage.") ) // Aggregation configuration flags @@ -106,28 +135,161 @@ var ( concurrentHPASyncs = flag.Int64("concurrent-hpa-syncs", 5, `The number of horizontal pod autoscaler objects that are allowed to sync concurrently. Larger number = more responsive MPA objects processing, but more CPU (and network) load.`) ) +// Post processors flags +var ( + // CPU as integer to benefit for CPU management Static Policy ( https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy ) + postProcessorCPUasInteger = flag.Bool("cpu-integer-post-processor-enabled", false, "Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental)") +) + const ( - discoveryResetPeriod time.Duration = 5 * time.Minute + // aggregateContainerStateGCInterval defines how often expired AggregateContainerStates are garbage collected. + aggregateContainerStateGCInterval = 1 * time.Hour + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. + scaleCacheLoopPeriod = 7 * time.Second + defaultResyncPeriod time.Duration = 10 * time.Minute + discoveryResetPeriod time.Duration = 5 * time.Minute ) func main() { + commonFlags := vpa_common.InitCommonFlags() klog.InitFlags(nil) + vpa_common.InitLoggingFlags() + + leaderElection := defaultLeaderElectionConfiguration() + componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) + kube_flag.InitFlags() klog.V(1).Infof("Multi-dimensional Pod Autoscaler %s Recommender: %v", common.MultidimPodAutoscalerVersion, recommenderName) - config := common.CreateKubeConfigOrDie(*kubeconfig, float32(*kubeApiQps), int(*kubeApiBurst)) - - vpa_model.InitializeAggregationsConfig(vpa_model.NewAggregationsConfig(*memoryAggregationInterval, *memoryAggregationIntervalCount, *memoryHistogramDecayHalfLife, *cpuHistogramDecayHalfLife, *oomBumpUpRatio, *oomMinBumpUp)) + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } healthCheck := metrics.NewHealthCheck(*metricsFetcherInterval * 5) - metrics.Initialize(*address, healthCheck) metrics_recommender.Register() metrics_quality.Register() + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + + if !leaderElection.LeaderElect { + run(healthCheck, commonFlags) + } else { + id, err := os.Hostname() + if err != nil { + klog.Fatalf("Unable to get hostname: %v", err) + } + id = id + "_" + string(uuid.NewUUID()) + + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + + lock, err := resourcelock.New( + leaderElection.ResourceLock, + leaderElection.ResourceNamespace, + leaderElection.ResourceName, + kubeClient.CoreV1(), + kubeClient.CoordinationV1(), + resourcelock.ResourceLockConfig{ + Identity: id, + }, + ) + if err != nil { + klog.Fatalf("Unable to create leader election lock: %v", err) + } + + leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaderElection.LeaseDuration.Duration, + RenewDeadline: leaderElection.RenewDeadline.Duration, + RetryPeriod: leaderElection.RetryPeriod.Duration, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + run(healthCheck, commonFlags) + }, + OnStoppedLeading: func() { + klog.Fatal("lost master") + }, + }, + }) + } +} + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 2 * time.Second +) + +func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { + return componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, + ResourceLock: resourcelock.LeasesResourceLock, + // This was changed from "vpa-recommender" to avoid conflicts with managed VPA deployments. + ResourceName: "mpa-recommender-lease", + ResourceNamespace: metav1.NamespaceSystem, + } +} + +func run(healthCheck *metrics.HealthCheck, commonFlag *vpa_common.CommonFlags) { + config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + clusterState := model.NewClusterState(aggregateContainerStateGCInterval) + factory := informers.NewSharedInformerFactoryWithOptions(kubeClient, defaultResyncPeriod, informers.WithNamespace(*ignoredMpaObjectNamespaces)) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) + podLister, oomObserver := input.NewPodListerAndOOMObserver(kubeClient, commonFlag.IgnoredVpaObjectNamespaces) + + vpa_model.InitializeAggregationsConfig(vpa_model.NewAggregationsConfig(*memoryAggregationInterval, *memoryAggregationIntervalCount, *memoryHistogramDecayHalfLife, *cpuHistogramDecayHalfLife, *oomBumpUpRatio, *oomMinBumpUp)) useCheckpoints := *storage != "prometheus" + var postProcessors []routines.RecommendationPostProcessor + if *postProcessorCPUasInteger { + postProcessors = append(postProcessors, &routines.IntegerCPUPostProcessor{}) + } + + // CappingPostProcessor, should always come in the last position for post-processing + postProcessors = append(postProcessors, &routines.CappingPostProcessor{}) + var source input_metrics.PodMetricsLister + if *useExternalMetrics { + resourceMetrics := map[apiv1.ResourceName]string{} + if externalCpuMetric != nil && *externalCpuMetric != "" { + resourceMetrics[apiv1.ResourceCPU] = *externalCpuMetric + } + if externalMemoryMetric != nil && *externalMemoryMetric != "" { + resourceMetrics[apiv1.ResourceMemory] = *externalMemoryMetric + } + externalClientOptions := &input_metrics.ExternalClientOptions{ResourceMetrics: resourceMetrics, ContainerNameLabel: *ctrNameLabel} + klog.V(1).InfoS("Using External Metrics", "options", externalClientOptions) + source = input_metrics.NewExternalClient(config, clusterState, *externalClientOptions) + } else { + klog.V(1).InfoS("Using Metrics Server") + source = input_metrics.NewPodMetricsesSource(resourceclient.NewForConfigOrDie(config)) + } + + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") + + clusterStateFeeder := input.ClusterStateFeederFactory{ + PodLister: podLister, + OOMObserver: oomObserver, + KubeClient: kubeClient, + MetricsClient: input_metrics.NewMetricsClient(source, *mpaObjectNamespace, "default-metrics-client"), + MpaCheckpointClient: mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1(), + MpaLister: mpa_api_util.NewMpasLister(mpa_clientset.NewForConfigOrDie(config), make(chan struct{}), *mpaObjectNamespace), + ClusterState: clusterState, + SelectorFetcher: target.NewMpaTargetSelectorFetcher(config, kubeClient, factory), + MemorySaveMode: *memorySaver, + ControllerFetcher: controllerFetcher, + RecommenderName: *recommenderName, + IgnoredNamespaces: ignoredNamespaces, + }.Make() + controllerFetcher.Start(context.Background(), scaleCacheLoopPeriod) + // For HPA. - kubeClient := kube_client.NewForConfigOrDie(config) // Use a discovery client capable of being refreshed. discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) if err != nil { @@ -154,7 +316,29 @@ func main() { external_metrics.NewForConfigOrDie(config), ) - recommender := routines.NewRecommender(config, *checkpointsGCInterval, useCheckpoints, *mpaObjectNamespace, *recommenderName, kubeClient.CoreV1(), metricsClient, *hpaSyncPeriod, *hpaDownscaleStabilizationWindow, *hpaTolerance, *hpaCPUInitializationPeriod, *hpaInitialReadinessDelay) + recommender := routines.RecommenderFactory{ + ClusterState: clusterState, + ClusterStateFeeder: clusterStateFeeder, + ControllerFetcher: controllerFetcher, + CheckpointWriter: checkpoint.NewCheckpointWriter(clusterState, mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1()), + MpaClient: mpa_clientset.NewForConfigOrDie(config).AutoscalingV1alpha1(), + SelectorFetcher: target.NewMpaTargetSelectorFetcher(config, kubeClient, factory), + PodResourceRecommender: logic.CreatePodResourceRecommender(), + RecommendationPostProcessors: postProcessors, + CheckpointsGCInterval: *checkpointsGCInterval, + UseCheckpoints: useCheckpoints, + + // HPA-related flags + EvtNamespacer: kubeClient.CoreV1(), + PodInformer: factory.Core().V1().Pods(), + MetricsClient: metricsClient, + ResyncPeriod: *hpaSyncPeriod, + DownscaleStabilisationWindow: *hpaDownscaleStabilizationWindow, + Tolerance: *hpaTolerance, + CpuInitializationPeriod: *hpaCPUInitializationPeriod, + DelayOfInitialReadinessStatus: *hpaInitialReadinessDelay, + }.Make() + klog.Infof("MPA Recommender created!") promQueryTimeout, err := time.ParseDuration(*queryTimeout) @@ -180,6 +364,10 @@ func main() { CtrNameLabel: *ctrNameLabel, CadvisorMetricsJobName: *prometheusJobName, Namespace: *mpaObjectNamespace, + PrometheusBasicAuthTransport: history.PrometheusBasicAuthTransport{ + Username: *username, + Password: *password, + }, } provider, err := history.NewPrometheusHistoryProvider(config) if err != nil { diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go index 409081a351b5..67661a59c728 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster.go @@ -17,6 +17,7 @@ limitations under the License. package model import ( + "context" "fmt" "time" @@ -356,10 +357,10 @@ func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID vpa // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) garbageCollectAggregateCollectionStates(now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { klog.V(1).Info("Garbage collection of AggregateCollectionStates triggered") keysToDelete := make([]vpa_model.AggregateStateKey, 0) - contributiveKeys := cluster.getContributiveAggregateStateKeys(controllerFetcher) + contributiveKeys := cluster.getContributiveAggregateStateKeys(ctx, controllerFetcher) for key, aggregateContainerState := range cluster.aggregateStateMap { isKeyContributive := contributiveKeys[key] if !isKeyContributive && isStateEmpty(aggregateContainerState) { @@ -390,21 +391,21 @@ func (cluster *ClusterState) garbageCollectAggregateCollectionStates(now time.Ti // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates(now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { if now.Sub(cluster.lastAggregateContainerStateGC) < cluster.gcInterval { return } - cluster.garbageCollectAggregateCollectionStates(now, controllerFetcher) + cluster.garbageCollectAggregateCollectionStates(ctx, now, controllerFetcher) cluster.lastAggregateContainerStateGC = now } -func (cluster *ClusterState) getContributiveAggregateStateKeys(controllerFetcher controllerfetcher.ControllerFetcher) map[vpa_model.AggregateStateKey]bool { +func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[vpa_model.AggregateStateKey]bool { contributiveKeys := map[vpa_model.AggregateStateKey]bool{} for _, pod := range cluster.Pods { // Pod is considered contributive in any of following situations: // 1) It is in active state - i.e. not PodSucceeded nor PodFailed. // 2) Its associated controller (e.g. Deployment) still exists. - podControllerExists := cluster.GetControllerForPodUnderVPA(pod, controllerFetcher) != nil + podControllerExists := cluster.GetControllerForPodUnderVPA(ctx, pod, controllerFetcher) != nil podActive := pod.Phase != apiv1.PodSucceeded && pod.Phase != apiv1.PodFailed if podActive || podControllerExists { for container := range pod.Containers { @@ -449,7 +450,7 @@ func (cluster *ClusterState) GetMatchingPods(mpa *Mpa) []vpa_model.PodID { } // GetControllerForPodUnderVPA returns controller associated with given Pod. Returns nil if Pod is not controlled by a VPA object. -func (cluster *ClusterState) GetControllerForPodUnderVPA(pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { +func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { controllingMPA := cluster.GetControllingMPA(pod) if controllingMPA != nil { controller := &controllerfetcher.ControllerKeyWithAPIVersion{ @@ -460,7 +461,7 @@ func (cluster *ClusterState) GetControllerForPodUnderVPA(pod *PodState, controll }, ApiVersion: controllingMPA.ScaleTargetRef.APIVersion, } - topLevelController, _ := controllerFetcher.FindTopMostWellKnownOrScalable(controller) + topLevelController, _ := controllerFetcher.FindTopMostWellKnownOrScalable(ctx, controller) return topLevelController } return nil diff --git a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go index b5bd9d13a190..81140b6dd3a9 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/model/cluster_test.go @@ -17,6 +17,7 @@ limitations under the License. package model import ( + "context" "fmt" "testing" "time" @@ -82,7 +83,7 @@ func (f *fakeControllerFetcher) Scales(namespace string) scale.ScaleInterface { return f.scaleNamespacer.Scales(namespace) } -func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(controller *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { +func (f *fakeControllerFetcher) FindTopMostWellKnownOrScalable(_ context.Context, controller *controllerfetcher.ControllerKeyWithAPIVersion) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { return f.key, f.err } @@ -112,6 +113,8 @@ func TestClusterAddSample(t *testing.T) { } func TestClusterGCAggregateContainerStateDeletesOld(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -127,7 +130,7 @@ func TestClusterGCAggregateContainerStateDeletesOld(t *testing.T) { assert.NotEmpty(t, mpa.aggregateContainerStates) // AggegateContainerState are valid for 8 days since last sample - cluster.garbageCollectAggregateCollectionStates(usageSample.MeasureStart.Add(9*24*time.Hour), testControllerFetcher) + cluster.garbageCollectAggregateCollectionStates(ctx, usageSample.MeasureStart.Add(9*24*time.Hour), testControllerFetcher) // AggegateContainerState should be deleted from both cluster and mpa assert.Empty(t, cluster.aggregateStateMap) @@ -135,6 +138,8 @@ func TestClusterGCAggregateContainerStateDeletesOld(t *testing.T) { } func TestClusterGCAggregateContainerStateDeletesOldEmpty(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -153,12 +158,12 @@ func TestClusterGCAggregateContainerStateDeletesOldEmpty(t *testing.T) { } // Verify empty aggregate states are not removed right away. - cluster.garbageCollectAggregateCollectionStates(creationTime.Add(1*time.Minute), testControllerFetcher) // AggegateContainerState should be deleted from both cluster and mpa + cluster.garbageCollectAggregateCollectionStates(ctx, creationTime.Add(1*time.Minute), testControllerFetcher) // AggegateContainerState should be deleted from both cluster and mpa assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) // AggegateContainerState are valid for 8 days since creation - cluster.garbageCollectAggregateCollectionStates(creationTime.Add(9*24*time.Hour), testControllerFetcher) + cluster.garbageCollectAggregateCollectionStates(ctx, creationTime.Add(9*24*time.Hour), testControllerFetcher) // AggegateContainerState should be deleted from both cluster and mpa assert.Empty(t, cluster.aggregateStateMap) @@ -166,6 +171,8 @@ func TestClusterGCAggregateContainerStateDeletesOldEmpty(t *testing.T) { } func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -182,14 +189,14 @@ func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) - cluster.garbageCollectAggregateCollectionStates(testTimestamp, controller) + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggegateContainerState should not be deleted as the pod is still active. assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded - cluster.garbageCollectAggregateCollectionStates(testTimestamp, controller) + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggegateContainerState should be empty as the pod is no longer active, controller is not alive // and there are no usage samples. @@ -198,6 +205,8 @@ func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t } func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -211,14 +220,14 @@ func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *te assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) - cluster.garbageCollectAggregateCollectionStates(testTimestamp, controller) + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggegateContainerState should not be deleted as the pod is still active. assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded - cluster.garbageCollectAggregateCollectionStates(testTimestamp, controller) + cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggegateContainerState should not be delated as the controller is still alive. assert.NotEmpty(t, cluster.aggregateStateMap) @@ -226,6 +235,8 @@ func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *te } func TestClusterGCAggregateContainerStateLeavesValid(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -241,13 +252,15 @@ func TestClusterGCAggregateContainerStateLeavesValid(t *testing.T) { assert.NotEmpty(t, mpa.aggregateContainerStates) // AggegateContainerState are valid for 8 days since last sample - cluster.garbageCollectAggregateCollectionStates(usageSample.MeasureStart.Add(7*24*time.Hour), testControllerFetcher) + cluster.garbageCollectAggregateCollectionStates(ctx, usageSample.MeasureStart.Add(7*24*time.Hour), testControllerFetcher) assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) } func TestAddSampleAfterAggregateContainerStateGCed(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) mpa := addTestMpa(cluster) @@ -268,7 +281,7 @@ func TestAddSampleAfterAggregateContainerStateGCed(t *testing.T) { // AggegateContainerState are invalid after 8 days since last sample gcTimestamp := usageSample.MeasureStart.Add(10 * 24 * time.Hour) - cluster.garbageCollectAggregateCollectionStates(gcTimestamp, testControllerFetcher) + cluster.garbageCollectAggregateCollectionStates(ctx, gcTimestamp, testControllerFetcher) assert.Empty(t, cluster.aggregateStateMap) assert.Empty(t, mpa.aggregateContainerStates) @@ -287,13 +300,15 @@ func TestAddSampleAfterAggregateContainerStateGCed(t *testing.T) { } func TestClusterGCRateLimiting(t *testing.T) { + ctx := context.Background() + // Create a pod with a single container. cluster := NewClusterState(testGcPeriod) usageSample := makeTestUsageSample() sampleExpireTime := usageSample.MeasureStart.Add(9 * 24 * time.Hour) // AggegateContainerState are valid for 8 days since last sample but this run // doesn't remove the sample, because we didn't add it yet. - cluster.RateLimitedGarbageCollectAggregateCollectionStates(sampleExpireTime, testControllerFetcher) + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime, testControllerFetcher) mpa := addTestMpa(cluster) addTestPod(cluster) assert.NoError(t, cluster.AddOrUpdateContainer(testContainerID, testRequest)) @@ -305,12 +320,12 @@ func TestClusterGCRateLimiting(t *testing.T) { // Sample is expired but this run doesn't remove it yet, because less than testGcPeriod // elapsed since the previous run. - cluster.RateLimitedGarbageCollectAggregateCollectionStates(sampleExpireTime.Add(testGcPeriod/2), testControllerFetcher) + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime.Add(testGcPeriod/2), testControllerFetcher) assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, mpa.aggregateContainerStates) // AggegateContainerState should be deleted from both cluster and mpa - cluster.RateLimitedGarbageCollectAggregateCollectionStates(sampleExpireTime.Add(2*testGcPeriod), testControllerFetcher) + cluster.RateLimitedGarbageCollectAggregateCollectionStates(ctx, sampleExpireTime.Add(2*testGcPeriod), testControllerFetcher) assert.Empty(t, cluster.aggregateStateMap) assert.Empty(t, mpa.aggregateContainerStates) } diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go new file mode 100644 index 000000000000..4d7e7a1a3576 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/capping_post_processor.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "k8s.io/klog/v2" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + vpa_utils "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa" +) + +// CappingPostProcessor ensure that the policy is applied to recommendation +// it applies policy for fields: MinAllowed and MaxAllowed +type CappingPostProcessor struct{} + +var _ RecommendationPostProcessor = &CappingPostProcessor{} + +// Process apply the capping post-processing to the recommendation. (use to be function getCappedRecommendation) +func (c CappingPostProcessor) Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources { + // TODO: maybe rename the vpa_utils.ApplyVPAPolicy to something that mention that it is doing capping only + cappedRecommendation, err := vpa_utils.ApplyVPAPolicy(recommendation, mpa.Spec.ResourcePolicy) + if err != nil { + klog.ErrorS(err, "Failed to apply policy for VPA", "vpa", klog.KObj(mpa)) + return recommendation + } + return cappedRecommendation +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go new file mode 100644 index 000000000000..1b34b9021235 --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "strings" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// IntegerCPUPostProcessor ensures that the recommendation delivers an integer value for CPU +// This is need for users who want to use CPU Management with static policy: https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy +type IntegerCPUPostProcessor struct{} + +const ( + // The user interface for that post processor is an annotation on the VPA object with the following format: + // vpa-post-processor.kubernetes.io/{containerName}_integerCPU=true + mpaPostProcessorPrefix = "mpa-post-processor.kubernetes.io/" + mpaPostProcessorIntegerCPUSuffix = "_integerCPU" + mpaPostProcessorIntegerCPUValue = "true" +) + +var _ RecommendationPostProcessor = &IntegerCPUPostProcessor{} + +// Process apply the capping post-processing to the recommendation. +// For this post processor the CPU value is rounded up to an integer +func (p *IntegerCPUPostProcessor) Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources { + + amendedRecommendation := recommendation.DeepCopy() + + for key, value := range mpa.Annotations { + containerName := extractContainerName(key, mpaPostProcessorPrefix, mpaPostProcessorIntegerCPUSuffix) + if containerName == "" || value != mpaPostProcessorIntegerCPUValue { + continue + } + + for _, r := range amendedRecommendation.ContainerRecommendations { + if r.ContainerName != containerName { + continue + } + setIntegerCPURecommendation(r.Target) + setIntegerCPURecommendation(r.LowerBound) + setIntegerCPURecommendation(r.UpperBound) + setIntegerCPURecommendation(r.UncappedTarget) + } + } + return amendedRecommendation +} + +func setIntegerCPURecommendation(recommendation apiv1.ResourceList) { + for resourceName, recommended := range recommendation { + if resourceName != apiv1.ResourceCPU { + continue + } + recommended.RoundUp(resource.Scale(0)) + recommendation[resourceName] = recommended + } +} + +// extractContainerName return the container name for the feature based on annotation key +// if the returned value is empty that means that the key does not match +func extractContainerName(key, prefix, suffix string) string { + if !strings.HasPrefix(key, prefix) { + return "" + } + if !strings.HasSuffix(key, suffix) { + return "" + } + + return key[len(prefix) : len(key)-len(suffix)] +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go new file mode 100644 index 000000000000..8e1c32c0960d --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/cpu_integer_post_processor_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test" +) + +func TestExtractContainerName(t *testing.T) { + tests := []struct { + name string + key string + prefix string + suffix string + want string + }{ + { + name: "empty", + key: "", + prefix: "", + suffix: "", + want: "", + }, + { + name: "no match", + key: "abc", + prefix: "z", + suffix: "x", + want: "", + }, + { + name: "match", + key: "abc", + prefix: "a", + suffix: "c", + want: "b", + }, + { + name: "real", + key: mpaPostProcessorPrefix + "kafka" + mpaPostProcessorIntegerCPUSuffix, + prefix: mpaPostProcessorPrefix, + suffix: mpaPostProcessorIntegerCPUSuffix, + want: "kafka", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, extractContainerName(tt.key, tt.prefix, tt.suffix), "extractContainerName(%v, %v, %v)", tt.key, tt.prefix, tt.suffix) + }) + } +} + +func TestIntegerCPUPostProcessor_Process(t *testing.T) { + tests := []struct { + name string + mpa *mpa_types.MultidimPodAutoscaler + recommendation *vpa_types.RecommendedPodResources + want *vpa_types.RecommendedPodResources + }{ + { + name: "No containers match", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + mpaPostProcessorPrefix + "container-other" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + }, + { + name: "2 containers, 1 matching only", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + mpaPostProcessorPrefix + "container1" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("9", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("8.2", "300Mi").GetContainerResources(), + }, + }, + }, + { + name: "2 containers, 2 matching", + mpa: &mpa_types.MultidimPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + mpaPostProcessorPrefix + "container1" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + mpaPostProcessorPrefix + "container2" + mpaPostProcessorIntegerCPUSuffix: mpaPostProcessorIntegerCPUValue, + }}}, + recommendation: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("8.6", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("5.2", "300Mi").GetContainerResources(), + }, + }, + want: &vpa_types.RecommendedPodResources{ + ContainerRecommendations: []vpa_types.RecommendedContainerResources{ + test.Recommendation().WithContainer("container1").WithTarget("9", "200Mi").GetContainerResources(), + test.Recommendation().WithContainer("container2").WithTarget("6", "300Mi").GetContainerResources(), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := IntegerCPUPostProcessor{} + got := c.Process(tt.mpa, tt.recommendation) + assert.True(t, equalRecommendedPodResources(tt.want, got), "Process(%v, %v, nil)", tt.mpa, tt.recommendation) + }) + } +} + +func equalRecommendedPodResources(a, b *vpa_types.RecommendedPodResources) bool { + if len(a.ContainerRecommendations) != len(b.ContainerRecommendations) { + return false + } + + for i := range a.ContainerRecommendations { + if !equalResourceList(a.ContainerRecommendations[i].LowerBound, b.ContainerRecommendations[i].LowerBound) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].Target, b.ContainerRecommendations[i].Target) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].UncappedTarget, b.ContainerRecommendations[i].UncappedTarget) { + return false + } + if !equalResourceList(a.ContainerRecommendations[i].UpperBound, b.ContainerRecommendations[i].UpperBound) { + return false + } + } + return true +} + +func equalResourceList(rla, rlb v1.ResourceList) bool { + if len(rla) != len(rlb) { + return false + } + for k := range rla { + q := rla[k] + if q.Cmp(rlb[k]) != 0 { + return false + } + } + for k := range rlb { + q := rlb[k] + if q.Cmp(rla[k]) != 0 { + return false + } + } + return true +} + +func TestSetIntegerCPURecommendation(t *testing.T) { + tests := []struct { + name string + recommendation v1.ResourceList + expectedRecommendation v1.ResourceList + }{ + { + name: "unchanged", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + { + name: "round up from 0.1", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8.1"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + { + name: "round up from 0.9", + recommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8.9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + expectedRecommendation: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("9"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setIntegerCPURecommendation(tt.recommendation) + assert.True(t, equalResourceList(tt.recommendation, tt.expectedRecommendation)) + }) + } +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go new file mode 100644 index 000000000000..352d3410c9bf --- /dev/null +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommendation_post_processor.go @@ -0,0 +1,27 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routines + +import ( + mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" + vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" +) + +// RecommendationPostProcessor can amend the recommendation according to the defined policies +type RecommendationPostProcessor interface { + Process(mpa *mpa_types.MultidimPodAutoscaler, recommendation *vpa_types.RecommendedPodResources) *vpa_types.RecommendedPodResources +} diff --git a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go index 59bbf1ef9e9d..7030fcc7a0a3 100644 --- a/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go +++ b/multidimensional-pod-autoscaler/pkg/recommender/routines/recommender.go @@ -110,6 +110,7 @@ type recommender struct { podResourceRecommender logic.PodResourceRecommender useCheckpoints bool lastAggregateContainerStateGC time.Time + recommendationPostProcessor []RecommendationPostProcessor // Fields for HPA. replicaCalc *hpa.ReplicaCalculator @@ -158,7 +159,14 @@ func (r *recommender) UpdateMPAs(ctx context.Context, vpaOrHpa string) { klog.V(4).Infof("Vertical scaling...") resources := r.podResourceRecommender.GetRecommendedPodResources(GetContainerNameToAggregateStateMap(mpa)) had := mpa.HasRecommendation() - mpa.UpdateRecommendation(getCappedRecommendation(mpa.ID, resources, observedMpa.Spec.ResourcePolicy)) + + listOfResourceRecommendation := logic.MapToListOfRecommendedContainerResources(resources) + + for _, postProcessor := range r.recommendationPostProcessor { + listOfResourceRecommendation = postProcessor.Process(observedMpa, listOfResourceRecommendation) + } + + mpa.UpdateRecommendation(listOfResourceRecommendation) klog.V(4).Infof("MPA %v recommendation updated: %v (%v)", key, resources, had) if mpa.HasRecommendation() && !had { metrics_recommender.ObserveRecommendationLatency(mpa.Created) @@ -221,10 +229,10 @@ func getCappedRecommendation(mpaID model.MpaID, resources logic.RecommendedPodRe for _, name := range containerNames { containerResources = append(containerResources, vpa_types.RecommendedContainerResources{ ContainerName: name, - Target: vpa_model.ResourcesAsResourceList(resources[name].Target), - LowerBound: vpa_model.ResourcesAsResourceList(resources[name].LowerBound), - UpperBound: vpa_model.ResourcesAsResourceList(resources[name].UpperBound), - UncappedTarget: vpa_model.ResourcesAsResourceList(resources[name].Target), + Target: vpa_model.ResourcesAsResourceList(resources[name].Target, logic.GetHumanizeMemory()), + LowerBound: vpa_model.ResourcesAsResourceList(resources[name].LowerBound, logic.GetHumanizeMemory()), + UpperBound: vpa_model.ResourcesAsResourceList(resources[name].UpperBound, logic.GetHumanizeMemory()), + UncappedTarget: vpa_model.ResourcesAsResourceList(resources[name].Target, logic.GetHumanizeMemory()), }) } recommendation := &vpa_types.RecommendedPodResources{ @@ -266,7 +274,7 @@ func (r *recommender) RunOnce(workers int, vpaOrHpa string) { klog.V(3).Infof("Recommender Run") defer klog.V(3).Infof("Shutting down MPA Recommender") - r.clusterStateFeeder.LoadMPAs() + r.clusterStateFeeder.LoadMPAs(ctx) timer.ObserveStep("LoadMPAs") r.clusterStateFeeder.LoadPods() @@ -282,7 +290,7 @@ func (r *recommender) RunOnce(workers int, vpaOrHpa string) { r.MaintainCheckpoints(ctx, *minCheckpointsPerRun) timer.ObserveStep("MaintainCheckpoints") - r.clusterState.RateLimitedGarbageCollectAggregateCollectionStates(time.Now(), r.controllerFetcher) + r.clusterState.RateLimitedGarbageCollectAggregateCollectionStates(ctx, time.Now(), r.controllerFetcher) timer.ObserveStep("GarbageCollect") klog.V(3).Infof("ClusterState is tracking %d aggregated container states", r.clusterState.StateMapSize()) } @@ -298,6 +306,8 @@ type RecommenderFactory struct { MpaClient mpa_api.MultidimPodAutoscalersGetter SelectorFetcher target.MpaTargetSelectorFetcher + RecommendationPostProcessors []RecommendationPostProcessor + CheckpointsGCInterval time.Duration UseCheckpoints bool @@ -332,6 +342,7 @@ func (c RecommenderFactory) Make() Recommender { mpaClient: c.MpaClient, selectorFetcher: c.SelectorFetcher, podResourceRecommender: c.PodResourceRecommender, + recommendationPostProcessor: c.RecommendationPostProcessors, lastAggregateContainerStateGC: time.Now(), lastCheckpointGC: time.Now(), diff --git a/multidimensional-pod-autoscaler/pkg/target/fetcher.go b/multidimensional-pod-autoscaler/pkg/target/fetcher.go index 700fe01e25df..b16f2879171e 100644 --- a/multidimensional-pod-autoscaler/pkg/target/fetcher.go +++ b/multidimensional-pod-autoscaler/pkg/target/fetcher.go @@ -50,7 +50,7 @@ const ( type MpaTargetSelectorFetcher interface { // Fetch returns a labelSelector used to gather Pods controlled by the given MPA. // If error is nil, the returned labelSelector is not nil. - Fetch(mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) + Fetch(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) // For updating the Deployments. GetRESTMappings(groupKind schema.GroupKind) ([]*apimeta.RESTMapping, error) @@ -120,7 +120,7 @@ type mpaTargetSelectorFetcher struct { informersMap map[wellKnownController]cache.SharedIndexInformer } -func (f *mpaTargetSelectorFetcher) Fetch(mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { +func (f *mpaTargetSelectorFetcher) Fetch(ctx context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { if mpa.Spec.ScaleTargetRef == nil { return nil, fmt.Errorf("scaleTargetRef not defined.") } @@ -141,7 +141,7 @@ func (f *mpaTargetSelectorFetcher) Fetch(mpa *mpa_types.MultidimPodAutoscaler) ( Kind: mpa.Spec.ScaleTargetRef.Kind, } - selector, err := f.getLabelSelectorFromResource(groupKind, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name) + selector, err := f.getLabelSelectorFromResource(ctx, groupKind, mpa.Namespace, mpa.Spec.ScaleTargetRef.Name) if err != nil { return nil, fmt.Errorf("Unhandled ScaleTargetRef %s / %s / %s, last error %v", mpa.Spec.ScaleTargetRef.APIVersion, mpa.Spec.ScaleTargetRef.Kind, mpa.Spec.ScaleTargetRef.Name, err) @@ -177,7 +177,7 @@ func getLabelSelector(informer cache.SharedIndexInformer, kind, namespace, name } func (f *mpaTargetSelectorFetcher) getLabelSelectorFromResource( - groupKind schema.GroupKind, namespace, name string, + ctx context.Context, groupKind schema.GroupKind, namespace, name string, ) (labels.Selector, error) { mappings, err := f.mapper.RESTMappings(groupKind) if err != nil { @@ -187,7 +187,7 @@ func (f *mpaTargetSelectorFetcher) getLabelSelectorFromResource( var lastError error for _, mapping := range mappings { groupResource := mapping.Resource.GroupResource() - scale, err := f.scaleNamespacer.Scales(namespace).Get(context.TODO(), groupResource, name, metav1.GetOptions{}) + scale, err := f.scaleNamespacer.Scales(namespace).Get(ctx, groupResource, name, metav1.GetOptions{}) if err == nil { if scale.Status.Selector == "" { return nil, fmt.Errorf("Resource %s/%s has an empty selector for scale sub-resource", namespace, name) diff --git a/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go b/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go index bc73136e9cc5..6821f9ae3b82 100644 --- a/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go +++ b/multidimensional-pod-autoscaler/pkg/target/mock/fetcher_mock.go @@ -17,34 +17,17 @@ limitations under the License. package mocktarget import ( + "context" + gomock "github.com/golang/mock/gomock" apimeta "k8s.io/apimachinery/pkg/api/meta" labels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" mpa_types "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1alpha1" - vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" "k8s.io/client-go/restmapper" "k8s.io/client-go/scale" ) -// MockVpaTargetSelectorFetcher is a mock of VpaTargetSelectorFetcher interface -type MockVpaTargetSelectorFetcher struct { - ctrl *gomock.Controller - recorder *_MockVpaTargetSelectorFetcherRecorder -} - -// Recorder for MockVpaTargetSelectorFetcher (not exported) -type _MockVpaTargetSelectorFetcherRecorder struct { - mock *MockVpaTargetSelectorFetcher -} - -// NewMockVpaTargetSelectorFetcher returns mock instance of a mock of VpaTargetSelectorFetcher -func NewMockVpaTargetSelectorFetcher(ctrl *gomock.Controller) *MockVpaTargetSelectorFetcher { - mock := &MockVpaTargetSelectorFetcher{ctrl: ctrl} - mock.recorder = &_MockVpaTargetSelectorFetcherRecorder{mock} - return mock -} - // MockMpaTargetSelectorFetcher is a mock of MpaTargetSelectorFetcher interface type MockMpaTargetSelectorFetcher struct { ctrl *gomock.Controller @@ -64,30 +47,13 @@ func NewMockMpaTargetSelectorFetcher(ctrl *gomock.Controller) *MockMpaTargetSele return mock } -// EXPECT enables configuring expectaions -func (_m *MockVpaTargetSelectorFetcher) EXPECT() *_MockVpaTargetSelectorFetcherRecorder { - return _m.recorder -} - -// Fetch enables configuring expectations on Fetch method -func (_m *MockVpaTargetSelectorFetcher) Fetch(vpa *vpa_types.VerticalPodAutoscaler) (labels.Selector, error) { - ret := _m.ctrl.Call(_m, "Fetch", vpa) - ret0, _ := ret[0].(labels.Selector) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -func (_mr *_MockVpaTargetSelectorFetcherRecorder) Fetch(arg0 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "Fetch", arg0) -} - // EXPECT enables configuring expectaions func (_m *MockMpaTargetSelectorFetcher) EXPECT() *_MockMpaTargetSelectorFetcherRecorder { return _m.recorder } // Fetch enables configuring expectations on Fetch method -func (_m *MockMpaTargetSelectorFetcher) Fetch(mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { +func (_m *MockMpaTargetSelectorFetcher) Fetch(_ context.Context, mpa *mpa_types.MultidimPodAutoscaler) (labels.Selector, error) { ret := _m.ctrl.Call(_m, "Fetch", mpa) ret0, _ := ret[0].(labels.Selector) ret1, _ := ret[1].(error) diff --git a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go index 5c72a0db8fe4..8b063f055cd2 100644 --- a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go +++ b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction.go @@ -44,7 +44,7 @@ const ( type PodsEvictionRestriction interface { // Evict sends eviction instruction to the api client. // Returns error if pod cannot be evicted or if client returned error. - Evict(pod *apiv1.Pod, eventRecorder record.EventRecorder) error + Evict(pod *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error // CanEvict checks if pod can be safely evicted CanEvict(pod *apiv1.Pod) bool } @@ -122,7 +122,7 @@ func (e *podsEvictionRestrictionImpl) CanEvict(pod *apiv1.Pod) bool { // Evict sends eviction instruction to api client. Returns error if pod cannot be evicted or if client returned error // Does not check if pod was actually evicted after eviction grace period. -func (e *podsEvictionRestrictionImpl) Evict(podToEvict *apiv1.Pod, eventRecorder record.EventRecorder) error { +func (e *podsEvictionRestrictionImpl) Evict(podToEvict *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error { cr, present := e.podToReplicaCreatorMap[getPodID(podToEvict)] if !present { return fmt.Errorf("pod not suitable for eviction %v : not in replicated pods map", podToEvict.Name) @@ -146,6 +146,9 @@ func (e *podsEvictionRestrictionImpl) Evict(podToEvict *apiv1.Pod, eventRecorder eventRecorder.Event(podToEvict, apiv1.EventTypeNormal, "EvictedByMPA", "Pod was evicted by MPA Updater to apply resource recommendation.") + eventRecorder.Event(mpa, apiv1.EventTypeNormal, "EvictedPod", + "MPA Updater evicted Pod "+podToEvict.Name+" to apply resource recommendation.") + if podToEvict.Status.Phase != apiv1.PodPending { singleGroupStats, present := e.creatorToSingleGroupStatsMap[cr] if !present { diff --git a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go index 1e2ad949da3e..56da4002e508 100644 --- a/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go +++ b/multidimensional-pod-autoscaler/pkg/updater/eviction/pods_eviction_restriction_test.go @@ -285,7 +285,7 @@ func TestEvictReplicatedByController(t *testing.T) { assert.Equalf(t, p.canEvict, eviction.CanEvict(p.pod), "TC %v - unexpected CanEvict result for pod-%v %#v", testCase.name, i, p.pod) } for i, p := range testCase.pods { - err := eviction.Evict(p.pod, test.FakeEventRecorder()) + err := eviction.Evict(p.pod, testCase.mpa, test.FakeEventRecorder()) if p.evictionSuccess { assert.NoErrorf(t, err, "TC %v - unexpected Evict result for pod-%v %#v", testCase.name, i, p.pod) } else { @@ -326,11 +326,11 @@ func TestEvictReplicatedByReplicaSet(t *testing.T) { } for _, pod := range pods[:2] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[2:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -365,11 +365,11 @@ func TestEvictReplicatedByStatefulSet(t *testing.T) { } for _, pod := range pods[:2] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[2:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -401,11 +401,11 @@ func TestEvictReplicatedByDaemonSet(t *testing.T) { assert.True(t, eviction.CanEvict(pod)) } for _, pod := range pods[:2] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[2:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -436,11 +436,11 @@ func TestEvictReplicatedByJob(t *testing.T) { } for _, pod := range pods[:2] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[2:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -475,7 +475,7 @@ func TestEvictTooFewReplicas(t *testing.T) { } for _, pod := range pods { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -511,11 +511,11 @@ func TestEvictionTolerance(t *testing.T) { } for _, pod := range pods[:4] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[4:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } @@ -551,11 +551,11 @@ func TestEvictAtLeastOne(t *testing.T) { } for _, pod := range pods[:1] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Nil(t, err, "Should evict with no error") } for _, pod := range pods[1:] { - err := eviction.Evict(pod, test.FakeEventRecorder()) + err := eviction.Evict(pod, getBasicMpa(), test.FakeEventRecorder()) assert.Error(t, err, "Error expected") } } diff --git a/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go b/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go index 7cb8467ae625..1a753b5150cb 100644 --- a/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go +++ b/multidimensional-pod-autoscaler/pkg/updater/logic/updater.go @@ -39,6 +39,7 @@ import ( "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/priority" mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" kube_client "k8s.io/client-go/kubernetes" @@ -73,6 +74,8 @@ type updater struct { selectorFetcher target.MpaTargetSelectorFetcher useAdmissionControllerStatus bool statusValidator status.Validator + controllerFetcher controllerfetcher.ControllerFetcher + ignoredNamespaces []string } // NewUpdater creates Updater with given configuration @@ -88,8 +91,10 @@ func NewUpdater( recommendationProcessor mpa_api_util.RecommendationProcessor, evictionAdmission priority.PodEvictionAdmission, selectorFetcher target.MpaTargetSelectorFetcher, + controllerFetcher controllerfetcher.ControllerFetcher, priorityProcessor priority.PriorityProcessor, namespace string, + ignoredNamespaces []string, ) (Updater, error) { evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst) factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction) @@ -106,12 +111,14 @@ func NewUpdater( evictionAdmission: evictionAdmission, priorityProcessor: priorityProcessor, selectorFetcher: selectorFetcher, + controllerFetcher: controllerFetcher, useAdmissionControllerStatus: useAdmissionControllerStatus, statusValidator: status.NewValidator( kubeClient, status.AdmissionControllerStatusName, statusNamespace, ), + ignoredNamespaces: ignoredNamespaces, }, nil } @@ -121,7 +128,7 @@ func (u *updater) RunOnce(ctx context.Context) { defer timer.ObserveTotal() if u.useAdmissionControllerStatus { - isValid, err := u.statusValidator.IsStatusValid(status.AdmissionControllerStatusTimeout) + isValid, err := u.statusValidator.IsStatusValid(ctx, status.AdmissionControllerStatusTimeout) if err != nil { klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err) return @@ -148,7 +155,7 @@ func (u *updater) RunOnce(ctx context.Context) { klog.V(3).Infof("skipping MPA object %v because its mode is not \"Recreate\" or \"Auto\"", mpa.Name) continue } - selector, err := u.selectorFetcher.Fetch(mpa) + selector, err := u.selectorFetcher.Fetch(ctx, mpa) if err != nil { klog.V(3).Infof("skipping MPA object %v because we cannot fetch selector", mpa.Name) continue @@ -179,7 +186,7 @@ func (u *updater) RunOnce(ctx context.Context) { controlledPods := make(map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) for _, pod := range allLivePods { - controllingMPA := mpa_api_util.GetControllingMPAForPod(pod, mpas) + controllingMPA := mpa_api_util.GetControllingMPAForPod(ctx, pod, mpas, u.controllerFetcher) if controllingMPA != nil { controlledPods[controllingMPA.Mpa] = append(controlledPods[controllingMPA.Mpa], pod) } @@ -226,7 +233,7 @@ func (u *updater) RunOnce(ctx context.Context) { return } klog.V(2).Infof("evicting pod %v", pod.Name) - evictErr := evictionLimiter.Evict(pod, u.eventRecorder) + evictErr := evictionLimiter.Evict(pod, mpa, u.eventRecorder) if evictErr != nil { klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr) } else { @@ -253,7 +260,7 @@ func (u *updater) RunOnceUpdatingDeployment(ctx context.Context) { defer timer.ObserveTotal() if u.useAdmissionControllerStatus { - isValid, err := u.statusValidator.IsStatusValid(status.AdmissionControllerStatusTimeout) + isValid, err := u.statusValidator.IsStatusValid(ctx, status.AdmissionControllerStatusTimeout) if err != nil { klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err) return @@ -280,7 +287,7 @@ func (u *updater) RunOnceUpdatingDeployment(ctx context.Context) { klog.V(3).Infof("skipping MPA object %v because its mode is not \"Recreate\" or \"Auto\"", mpa.Name) continue } - selector, err := u.selectorFetcher.Fetch(mpa) + selector, err := u.selectorFetcher.Fetch(ctx, mpa) if err != nil { klog.V(3).Infof("skipping MPA object %v because we cannot fetch selector", mpa.Name) continue @@ -355,7 +362,7 @@ func (u *updater) RunOnceUpdatingDeployment(ctx context.Context) { controlledPods := make(map[*mpa_types.MultidimPodAutoscaler][]*apiv1.Pod) for _, pod := range allLivePods { - controllingMPA := mpa_api_util.GetControllingMPAForPod(pod, mpas) + controllingMPA := mpa_api_util.GetControllingMPAForPod(ctx, pod, mpas, u.controllerFetcher) if controllingMPA != nil { controlledPods[controllingMPA.Mpa] = append(controlledPods[controllingMPA.Mpa], pod) } @@ -402,7 +409,7 @@ func (u *updater) RunOnceUpdatingDeployment(ctx context.Context) { return } klog.V(2).Infof("evicting pod %v", pod.Name) - evictErr := evictionLimiter.Evict(pod, u.eventRecorder) + evictErr := evictionLimiter.Evict(pod, mpa, u.eventRecorder) if evictErr != nil { klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr) } else { diff --git a/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go b/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go index 3b41ba8b385d..9ac0cabe725f 100644 --- a/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go +++ b/multidimensional-pod-autoscaler/pkg/updater/logic/updater_test.go @@ -143,7 +143,7 @@ func testRunOnceBase( }, } pods := make([]*apiv1.Pod, livePods) - eviction := &test.PodsEvictionRestrictionMock{} + eviction := &mpa_test.PodsEvictionRestrictionMock{} for i := range pods { pods[i] = test.Pod().WithName("test_"+strconv.Itoa(i)). @@ -196,7 +196,7 @@ func testRunOnceBase( } func TestRunOnceNotingToProcess(t *testing.T) { - eviction := &test.PodsEvictionRestrictionMock{} + eviction := &mpa_test.PodsEvictionRestrictionMock{} factory := &fakeEvictFactory{eviction} mpaLister := &mpa_test.MultidimPodAutoscalerListerMock{} podLister := &test.PodListerMock{} @@ -247,6 +247,6 @@ func newFakeValidator(isValid bool) status.Validator { return &fakeValidator{isValid} } -func (f *fakeValidator) IsStatusValid(statusTimeout time.Duration) (bool, error) { +func (f *fakeValidator) IsStatusValid(ctx context.Context, statusTimeout time.Duration) (bool, error) { return f.isValid, nil } diff --git a/multidimensional-pod-autoscaler/pkg/updater/main.go b/multidimensional-pod-autoscaler/pkg/updater/main.go index afb8a68ad0b0..5f4baa378954 100644 --- a/multidimensional-pod-autoscaler/pkg/updater/main.go +++ b/multidimensional-pod-autoscaler/pkg/updater/main.go @@ -20,22 +20,33 @@ import ( "context" "flag" "os" + "strings" "time" + "github.com/spf13/pflag" apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/autoscaler/multidimensional-pod-autoscaler/common" mpa_clientset "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/target" updater "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/logic" "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/updater/priority" mpa_api_util "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/utils/mpa" + vpa_common "k8s.io/autoscaler/vertical-pod-autoscaler/common" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics" metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/server" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status" "k8s.io/client-go/informers" kube_client "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" kube_flag "k8s.io/component-base/cli/flag" + componentbaseconfig "k8s.io/component-base/config" + componentbaseoptions "k8s.io/component-base/config/options" "k8s.io/klog/v2" ) @@ -55,44 +66,125 @@ var ( evictionRateBurst = flag.Int("eviction-rate-burst", 1, `Burst of pods that can be evicted.`) - address = flag.String("address", ":8943", "The address to expose Prometheus metrics.") - kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") - kubeApiQps = flag.Float64("kube-api-qps", 5.0, `QPS limit when making requests to Kubernetes apiserver`) - kubeApiBurst = flag.Float64("kube-api-burst", 10.0, `QPS burst limit when making requests to Kubernetes apiserver`) + address = flag.String("address", ":8943", "The address to expose Prometheus metrics.") useAdmissionControllerStatus = flag.Bool("use-admission-controller-status", true, "If true, updater will only evict pods when admission controller status is valid.") - namespace = os.Getenv("NAMESPACE") - mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + namespace = os.Getenv("NAMESPACE") + mpaObjectNamespace = flag.String("mpa-object-namespace", apiv1.NamespaceAll, "Namespace to search for MPA objects. Empty means all namespaces will be used.") + ignoredMpaObjectNamespaces = flag.String("ignored-mpa-object-namespaces", "", "Comma separated list of namespaces to ignore when searching for MPA objects. Empty means no namespaces will be ignored.") ) -const defaultResyncPeriod time.Duration = 10 * time.Minute +const ( + defaultResyncPeriod time.Duration = 10 * time.Minute + scaleCacheEntryLifetime time.Duration = time.Hour + scaleCacheEntryFreshnessTime time.Duration = 10 * time.Minute + scaleCacheEntryJitterFactor float64 = 1. +) func main() { + commonFlags := vpa_common.InitCommonFlags() klog.InitFlags(nil) + vpa_common.InitLoggingFlags() + + leaderElection := defaultLeaderElectionConfiguration() + componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) + kube_flag.InitFlags() klog.V(1).Infof("Multidimensional Pod Autoscaler %s Updater", common.MultidimPodAutoscalerVersion) + if len(*mpaObjectNamespace) > 0 && len(*ignoredMpaObjectNamespaces) > 0 { + klog.Fatalf("--mpa-object-namespace and --ignored-mpa-object-namespaces are mutually exclusive and can't be set together.") + } + healthCheck := metrics.NewHealthCheck(*updaterInterval * 5) - metrics.Initialize(*address, healthCheck) + server.Initialize(&commonFlags.EnableProfiling, healthCheck, address) + metrics_updater.Register() - config := common.CreateKubeConfigOrDie(*kubeconfig, float32(*kubeApiQps), int(*kubeApiBurst)) + if !leaderElection.LeaderElect { + run(healthCheck, commonFlags) + } else { + id, err := os.Hostname() + if err != nil { + klog.Fatalf("Unable to get hostname: %v", err) + } + id = id + "_" + string(uuid.NewUUID()) + + config := common.CreateKubeConfigOrDie(commonFlags.KubeConfig, float32(commonFlags.KubeApiQps), int(commonFlags.KubeApiBurst)) + kubeClient := kube_client.NewForConfigOrDie(config) + + lock, err := resourcelock.New( + leaderElection.ResourceLock, + leaderElection.ResourceNamespace, + leaderElection.ResourceName, + kubeClient.CoreV1(), + kubeClient.CoordinationV1(), + resourcelock.ResourceLockConfig{ + Identity: id, + }, + ) + if err != nil { + klog.Fatalf("Unable to create leader election lock: %v", err) + } + + leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaderElection.LeaseDuration.Duration, + RenewDeadline: leaderElection.RenewDeadline.Duration, + RetryPeriod: leaderElection.RetryPeriod.Duration, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + run(healthCheck, commonFlags) + }, + OnStoppedLeading: func() { + klog.Fatal("lost master") + }, + }, + }) + } +} + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 2 * time.Second +) + +func defaultLeaderElectionConfiguration() componentbaseconfig.LeaderElectionConfiguration { + return componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: defaultLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: defaultRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: defaultRetryPeriod}, + ResourceLock: resourcelock.LeasesResourceLock, + ResourceName: "mpa-updater", + ResourceNamespace: metav1.NamespaceSystem, + } +} + +func run(healthCheck *metrics.HealthCheck, commonFlag *vpa_common.CommonFlags) { + config := common.CreateKubeConfigOrDie(commonFlag.KubeConfig, float32(commonFlag.KubeApiQps), int(commonFlag.KubeApiBurst)) kubeClient := kube_client.NewForConfigOrDie(config) mpaClient := mpa_clientset.NewForConfigOrDie(config) factory := informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod) targetSelectorFetcher := target.NewMpaTargetSelectorFetcher(config, kubeClient, factory) + controllerFetcher := controllerfetcher.NewControllerFetcher(config, kubeClient, factory, scaleCacheEntryFreshnessTime, scaleCacheEntryLifetime, scaleCacheEntryJitterFactor) var limitRangeCalculator limitrange.LimitRangeCalculator limitRangeCalculator, err := limitrange.NewLimitsRangeCalculator(factory) if err != nil { - klog.Errorf("Failed to create limitRangeCalculator, falling back to not checking limits. Error message: %s", err) + klog.ErrorS(err, "Failed to create limitRangeCalculator, falling back to not checking limits") limitRangeCalculator = limitrange.NewNoopLimitsCalculator() } admissionControllerStatusNamespace := status.AdmissionControllerStatusNamespace if namespace != "" { admissionControllerStatusNamespace = namespace } + + ignoredNamespaces := strings.Split(*ignoredMpaObjectNamespaces, ",") + // TODO: use SharedInformerFactory in updater updater, err := updater.NewUpdater( kubeClient, @@ -106,14 +198,18 @@ func main() { mpa_api_util.NewCappingRecommendationProcessor(limitRangeCalculator), nil, targetSelectorFetcher, + controllerFetcher, priority.NewProcessor(), *mpaObjectNamespace, + ignoredNamespaces, ) if err != nil { klog.Fatalf("Failed to create updater: %v", err) - } else { - klog.V(1).Infof("Updater created!") } + + // Start updating health check endpoint. + healthCheck.StartMonitoring() + ticker := time.Tick(*updaterInterval) for range ticker { ctx, cancel := context.WithTimeout(context.Background(), *updaterInterval) diff --git a/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go b/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go index 2845e0c363cb..2b8f99a9986e 100644 --- a/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go +++ b/multidimensional-pod-autoscaler/pkg/utils/mpa/api.go @@ -35,6 +35,7 @@ import ( mpa_api "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1alpha1" mpa_lister "k8s.io/autoscaler/multidimensional-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1alpha1" vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" ) @@ -51,14 +52,29 @@ type patchRecord struct { Value interface{} `json:"value"` } -func patchMpa(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, patches []patchRecord) (result *mpa_types.MultidimPodAutoscaler, err error) { +func patchMpaStatus(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, patches []patchRecord) (result *mpa_types.MultidimPodAutoscaler, err error) { bytes, err := json.Marshal(patches) if err != nil { klog.Errorf("Cannot marshal MPA status patches %+v. Reason: %+v", patches, err) return } - return mpaClient.Patch(context.TODO(), mpaName, types.JSONPatchType, bytes, meta.PatchOptions{}) + return mpaClient.Patch(context.TODO(), mpaName, types.JSONPatchType, bytes, meta.PatchOptions{}, "status") +} + +// UpdateMpaStatusIfNeeded updates the status field of the MPA API object. +func UpdateMpaStatusIfNeeded(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, newStatus, + oldStatus *mpa_types.MultidimPodAutoscalerStatus) (result *mpa_types.MultidimPodAutoscaler, err error) { + patches := []patchRecord{{ + Op: "add", + Path: "/status", + Value: *newStatus, + }} + + if !apiequality.Semantic.DeepEqual(*oldStatus, *newStatus) { + return patchMpaStatus(mpaClient, mpaName, patches) + } + return nil, nil } // NewMpasLister returns MultidimPodAutoscalerLister configured to fetch all MPA objects from @@ -81,54 +97,6 @@ func NewMpasLister(mpaClient *mpa_clientset.Clientset, stopChannel <-chan struct return mpaLister } -// CreateOrUpdateMpaCheckpoint updates the status field of the MPA Checkpoint API object. -// If object doesn't exits it is created. -func CreateOrUpdateMpaCheckpoint(mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointInterface, - mpaCheckpoint *mpa_types.MultidimPodAutoscalerCheckpoint) error { - patches := make([]patchRecord, 0) - patches = append(patches, patchRecord{ - Op: "replace", - Path: "/status", - Value: mpaCheckpoint.Status, - }) - bytes, err := json.Marshal(patches) - if err != nil { - return fmt.Errorf("Cannot marshal MPA checkpoint status patches %+v. Reason: %+v", patches, err) - } - _, err = mpaCheckpointClient.Patch(context.TODO(), mpaCheckpoint.ObjectMeta.Name, types.JSONPatchType, bytes, meta.PatchOptions{}) - if err != nil && strings.Contains(err.Error(), fmt.Sprintf("\"%s\" not found", mpaCheckpoint.ObjectMeta.Name)) { - _, err = mpaCheckpointClient.Create(context.TODO(), mpaCheckpoint, meta.CreateOptions{}) - } - if err != nil { - return fmt.Errorf("Cannot save checkpoint for mpa %v container %v. Reason: %+v", mpaCheckpoint.ObjectMeta.Name, mpaCheckpoint.Spec.ContainerName, err) - } - return nil -} - -// UpdateMpaStatusIfNeeded updates the status field of the MPA API object. -func UpdateMpaStatusIfNeeded(mpaClient mpa_api.MultidimPodAutoscalerInterface, mpaName string, newStatus, - oldStatus *mpa_types.MultidimPodAutoscalerStatus) (result *mpa_types.MultidimPodAutoscaler, err error) { - patches := []patchRecord{{ - Op: "add", - Path: "/status", - Value: *newStatus, - }} - - if !apiequality.Semantic.DeepEqual(*oldStatus, *newStatus) { - return patchMpa(mpaClient, mpaName, patches) - } - return nil, nil -} - -// GetUpdateMode returns the updatePolicy.updateMode for a given MPA. -// If the mode is not specified it returns the default (UpdateModeAuto). -func GetUpdateMode(mpa *mpa_types.MultidimPodAutoscaler) vpa_types.UpdateMode { - if mpa.Spec.Policy == nil || mpa.Spec.Policy.UpdateMode == nil || *mpa.Spec.Policy.UpdateMode == "" { - return vpa_types.UpdateModeAuto - } - return *mpa.Spec.Policy.UpdateMode -} - // PodMatchesMPA returns true iff the mpaWithSelector matches the Pod. func PodMatchesMPA(pod *core.Pod, mpaWithSelector *MpaWithSelector) bool { return PodLabelsMatchMPA(pod.Namespace, labels.Set(pod.GetLabels()), mpaWithSelector.Mpa.Namespace, mpaWithSelector.Selector) @@ -142,8 +110,8 @@ func PodLabelsMatchMPA(podNamespace string, labels labels.Set, mpaNamespace stri return mpaSelector.Matches(labels) } -// stronger returns true iff a is before b in the order to control a Pod (that matches both MPAs). -func stronger(a, b *mpa_types.MultidimPodAutoscaler) bool { +// Stronger returns true iff a is before b in the order to control a Pod (that matches both MPAs). +func Stronger(a, b *mpa_types.MultidimPodAutoscaler) bool { // Assume a is not nil and each valid object is before nil object. if b == nil { return true @@ -160,15 +128,91 @@ func stronger(a, b *mpa_types.MultidimPodAutoscaler) bool { } // GetControllingMPAForPod chooses the earliest created MPA from the input list that matches the given Pod. -func GetControllingMPAForPod(pod *core.Pod, mpas []*MpaWithSelector) *MpaWithSelector { +func GetControllingMPAForPod(ctx context.Context, pod *core.Pod, mpas []*MpaWithSelector, ctrlFetcher controllerfetcher.ControllerFetcher) *MpaWithSelector { + + parentController, err := FindParentControllerForPod(ctx, pod, ctrlFetcher) + if err != nil { + klog.ErrorS(err, "Failed to get parent controller for pod", "pod", klog.KObj(pod)) + return nil + } + if parentController == nil { + return nil + } + var controlling *MpaWithSelector var controllingMpa *mpa_types.MultidimPodAutoscaler // Choose the strongest MPA from the ones that match this Pod. for _, mpaWithSelector := range mpas { - if PodMatchesMPA(pod, mpaWithSelector) && stronger(mpaWithSelector.Mpa, controllingMpa) { + if mpaWithSelector.Mpa.Spec.ScaleTargetRef == nil { + klog.V(5).InfoS("Skipping MPA object because scaleTargetRef is not defined.", "mpa", klog.KObj(mpaWithSelector.Mpa)) + continue + } + if mpaWithSelector.Mpa.Spec.ScaleTargetRef.Kind != parentController.Kind || + mpaWithSelector.Mpa.Namespace != parentController.Namespace || + mpaWithSelector.Mpa.Spec.ScaleTargetRef.Name != parentController.Name { + continue // This pod is not associated to the right controller + } + if PodMatchesMPA(pod, mpaWithSelector) && Stronger(mpaWithSelector.Mpa, controllingMpa) { controlling = mpaWithSelector controllingMpa = controlling.Mpa } } return controlling } + +// FindParentControllerForPod returns the parent controller (topmost well-known or scalable controller) for the given Pod. +func FindParentControllerForPod(ctx context.Context, pod *core.Pod, ctrlFetcher controllerfetcher.ControllerFetcher) (*controllerfetcher.ControllerKeyWithAPIVersion, error) { + var ownerRefrence *meta.OwnerReference + for i := range pod.OwnerReferences { + r := pod.OwnerReferences[i] + if r.Controller != nil && *r.Controller { + ownerRefrence = &r + } + } + if ownerRefrence == nil { + // If the pod has no ownerReference, it cannot be under a VPA. + return nil, nil + } + k := &controllerfetcher.ControllerKeyWithAPIVersion{ + ControllerKey: controllerfetcher.ControllerKey{ + Namespace: pod.Namespace, + Kind: ownerRefrence.Kind, + Name: ownerRefrence.Name, + }, + ApiVersion: ownerRefrence.APIVersion, + } + return ctrlFetcher.FindTopMostWellKnownOrScalable(ctx, k) +} + +// GetUpdateMode returns the updatePolicy.updateMode for a given MPA. +// If the mode is not specified it returns the default (UpdateModeAuto). +func GetUpdateMode(mpa *mpa_types.MultidimPodAutoscaler) vpa_types.UpdateMode { + if mpa.Spec.Policy == nil || mpa.Spec.Policy.UpdateMode == nil || *mpa.Spec.Policy.UpdateMode == "" { + return vpa_types.UpdateModeAuto + } + return *mpa.Spec.Policy.UpdateMode +} + +// CreateOrUpdateMpaCheckpoint updates the status field of the MPA Checkpoint API object. +// If object doesn't exits it is created. +func CreateOrUpdateMpaCheckpoint(mpaCheckpointClient mpa_api.MultidimPodAutoscalerCheckpointInterface, + mpaCheckpoint *mpa_types.MultidimPodAutoscalerCheckpoint) error { + patches := make([]patchRecord, 0) + patches = append(patches, patchRecord{ + Op: "replace", + Path: "/status", + Value: mpaCheckpoint.Status, + }) + bytes, err := json.Marshal(patches) + if err != nil { + return fmt.Errorf("Cannot marshal MPA checkpoint status patches %+v. Reason: %+v", patches, err) + } + _, err = mpaCheckpointClient.Patch(context.TODO(), mpaCheckpoint.ObjectMeta.Name, types.JSONPatchType, bytes, meta.PatchOptions{}) + if err != nil && strings.Contains(err.Error(), fmt.Sprintf("\"%s\" not found", mpaCheckpoint.ObjectMeta.Name)) { + _, err = mpaCheckpointClient.Create(context.TODO(), mpaCheckpoint, meta.CreateOptions{}) + } + if err != nil { + return fmt.Errorf("Cannot save checkpoint for mpa %v container %v. Reason: %+v", mpaCheckpoint.ObjectMeta.Name, mpaCheckpoint.Spec.ContainerName, err) + } + return nil +} diff --git a/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go b/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go index a777edcdf90f..17e3e6c1878d 100644 --- a/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go +++ b/multidimensional-pod-autoscaler/pkg/utils/test/test_utils.go @@ -127,7 +127,7 @@ type PodsEvictionRestrictionMock struct { } // Evict is a mock implementation of PodsEvictionRestriction.Evict -func (m *PodsEvictionRestrictionMock) Evict(pod *apiv1.Pod, eventRecorder record.EventRecorder) error { +func (m *PodsEvictionRestrictionMock) Evict(pod *apiv1.Pod, mpa *mpa_types.MultidimPodAutoscaler, eventRecorder record.EventRecorder) error { args := m.Called(pod, eventRecorder) return args.Error(0) } diff --git a/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go b/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go index 8aaf9cbd5e91..a38bfe6dfb77 100644 --- a/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go +++ b/vertical-pod-autoscaler/pkg/recommender/logic/recommender.go @@ -37,6 +37,11 @@ var ( humanizeMemory = flag.Bool("humanize-memory", false, "Convert memory values in recommendations to the highest appropriate SI unit with up to 2 decimal places for better readability.") ) +// GetHumanizeMemory returns the value of the HumanizeMemory flag. +func GetHumanizeMemory() bool { + return *humanizeMemory +} + // PodResourceRecommender computes resource recommendation for a Vpa object. type PodResourceRecommender interface { GetRecommendedPodResources(containerNameToAggregateStateMap model.ContainerNameToAggregateStateMap) RecommendedPodResources