diff --git a/.ci/tests/integration/e2e_with_tls.yaml b/.ci/tests/integration/e2e_with_tls.yaml index afb6ed10..88a90971 100644 --- a/.ci/tests/integration/e2e_with_tls.yaml +++ b/.ci/tests/integration/e2e_with_tls.yaml @@ -24,6 +24,17 @@ setup: label-selector: app=cert-manager for: condition=Ready + - name: setup vpa + command: | + helm repo add cowboysysop https://cowboysysop.github.io/charts/ + helm repo update + helm install vpa cowboysysop/vertical-pod-autoscaler + wait: + - namespace: default + resource: pod + label-selector: app.kubernetes.io/name=vertical-pod-autoscaler + for: condition=Ready + - name: install pulsar cluster command: | helm repo add streamnative https://charts.streamnative.io @@ -37,6 +48,11 @@ setup: helm dependency update charts/pulsar helm install ${PULSAR_RELEASE_NAME} --set initialize=true --values ../.ci/clusters/values_skywalking_e2e_cluster_with_tls.yaml charts/pulsar + - name: install metrics service + command: | + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + kubectl patch -n kube-system deployment metrics-server --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' + - name: wait for pulsar cluster ready command: | echo "wait until pulsar init job is completed" diff --git a/controllers/common.go b/controllers/common.go index b2c9b4a5..3a80311d 100644 --- a/controllers/common.go +++ b/controllers/common.go @@ -117,11 +117,9 @@ func observeVPA(ctx context.Context, r client.Reader, name types.NamespacedName, } // compare exists VPA with new Spec - updatePolicy := vpaSpec.UpdatePolicy - spec.UpdateVPAUpdatePolicy(updatePolicy, vpaSpec.ResourceUnit) - resourcePolicy := vpaSpec.ResourcePolicy + updatePolicy := spec.UpdateVPAUpdatePolicy(vpaSpec.UpdatePolicy, vpaSpec.ResourceUnit) containerName := spec.GetVPAContainerName(&vpa.ObjectMeta) - spec.UpdateResourcePolicy(resourcePolicy, containerName) + resourcePolicy := spec.UpdateResourcePolicy(vpaSpec.ResourcePolicy, containerName) if !reflect.DeepEqual(updatePolicy, vpa.Spec.UpdatePolicy) || !reflect.DeepEqual(resourcePolicy, vpa.Spec.ResourcePolicy) { condition.Status = metav1.ConditionFalse diff --git a/controllers/spec/vpa.go b/controllers/spec/vpa.go index f95d0799..da27175b 100644 --- a/controllers/spec/vpa.go +++ b/controllers/spec/vpa.go @@ -28,13 +28,11 @@ import ( func MakeVPA(objectMeta *metav1.ObjectMeta, targetRef *autov2.CrossVersionObjectReference, vpa *v1alpha1.VPASpec) *vpav1.VerticalPodAutoscaler { containerName := GetVPAContainerName(objectMeta) - updatePolicy := vpa.UpdatePolicy - UpdateVPAUpdatePolicy(updatePolicy, vpa.ResourceUnit) + updatePolicy := UpdateVPAUpdatePolicy(vpa.UpdatePolicy, vpa.ResourceUnit) if vpa.ResourceUnit != nil { objectMeta.Labels[LabelCustomResourceUnit] = "true" } - resourcePolicy := vpa.ResourcePolicy - UpdateResourcePolicy(resourcePolicy, containerName) + resourcePolicy := UpdateResourcePolicy(vpa.ResourcePolicy, containerName) return &vpav1.VerticalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ @@ -67,19 +65,24 @@ func GetVPAContainerName(objectMeta *metav1.ObjectMeta) string { return containerName } -func UpdateVPAUpdatePolicy(updatePolicy *vpav1.PodUpdatePolicy, resourceUnit *v1alpha1.ResourceUnit) { - if updatePolicy != nil && resourceUnit != nil { +func UpdateVPAUpdatePolicy(updatePolicy *vpav1.PodUpdatePolicy, resourceUnit *v1alpha1.ResourceUnit) *vpav1.PodUpdatePolicy { + resultUpdatePolicy := updatePolicy + if resourceUnit != nil { + if resultUpdatePolicy == nil { + resultUpdatePolicy = &vpav1.PodUpdatePolicy{} + } off := vpav1.UpdateModeOff - updatePolicy.UpdateMode = &off + resultUpdatePolicy.UpdateMode = &off } + return resultUpdatePolicy } -func UpdateResourcePolicy(resourcePolicy *vpav1.PodResourcePolicy, containerName string) { +func UpdateResourcePolicy(resourcePolicy *vpav1.PodResourcePolicy, containerName string) *vpav1.PodResourcePolicy { + var containerPolicies []vpav1.ContainerResourcePolicy + containerScalingMode := vpav1.ContainerScalingModeAuto if resourcePolicy != nil { - var containerPolicies []vpav1.ContainerResourcePolicy - containerScalingMode := vpav1.ContainerScalingModeAuto if resourcePolicy.ContainerPolicies == nil { - resourcePolicy.ContainerPolicies = []vpav1.ContainerResourcePolicy{} + containerPolicies = []vpav1.ContainerResourcePolicy{} } for _, policy := range resourcePolicy.ContainerPolicies { if policy.ContainerName == containerName { @@ -88,15 +91,21 @@ func UpdateResourcePolicy(resourcePolicy *vpav1.PodResourcePolicy, containerName break } } - // if resource policy is not set, set the default policy, so the vpa policy won't be applied to other containers - if len(containerPolicies) == 0 { - containerPolicies = []vpav1.ContainerResourcePolicy{ - { - ContainerName: containerName, - Mode: &containerScalingMode, - }, - } + } + + // if resource policy is not set, set the default policy, so the vpa policy won't be applied to other containers + if len(containerPolicies) == 0 { + containerPolicies = []vpav1.ContainerResourcePolicy{ + { + ContainerName: containerName, + Mode: &containerScalingMode, + }, } - resourcePolicy.ContainerPolicies = containerPolicies } + resultResourcePolicy := resourcePolicy + if resultResourcePolicy == nil { + resultResourcePolicy = &vpav1.PodResourcePolicy{} + } + resultResourcePolicy.ContainerPolicies = containerPolicies + return resultResourcePolicy } diff --git a/controllers/spec/vpa_test.go b/controllers/spec/vpa_test.go index cf8ec3aa..51d5de78 100644 --- a/controllers/spec/vpa_test.go +++ b/controllers/spec/vpa_test.go @@ -32,7 +32,11 @@ import ( func TestMakeVPA(t *testing.T) { mode := vpav1.UpdateModeAuto + off := vpav1.UpdateModeOff + containerMode := vpav1.ContainerScalingModeAuto + containerOffMode := vpav1.ContainerScalingModeOff controlledValues := vpav1.ContainerControlledValuesRequestsAndLimits + var minReplicas int32 = 3 type args struct { objectMeta *metav1.ObjectMeta targetRef *autov2.CrossVersionObjectReference @@ -48,6 +52,9 @@ func TestMakeVPA(t *testing.T) { args: args{ objectMeta: &metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "function", + }, }, targetRef: &autov2.CrossVersionObjectReference{ APIVersion: "apps/v1", @@ -60,7 +67,7 @@ func TestMakeVPA(t *testing.T) { }, ResourcePolicy: &vpav1.PodResourcePolicy{ ContainerPolicies: []vpav1.ContainerResourcePolicy{{ - ContainerName: "test-container", + ContainerName: "pulsar-function", MinAllowed: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), v1.ResourceMemory: resource.MustParse("100Mi"), @@ -80,6 +87,9 @@ func TestMakeVPA(t *testing.T) { want: &vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "function", + }, }, TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling.k8s.io/v1", @@ -97,7 +107,7 @@ func TestMakeVPA(t *testing.T) { ResourcePolicy: &vpav1.PodResourcePolicy{ ContainerPolicies: []vpav1.ContainerResourcePolicy{ { - ContainerName: "test-container", + ContainerName: "pulsar-function", MinAllowed: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), v1.ResourceMemory: resource.MustParse("100Mi"), @@ -110,6 +120,7 @@ func TestMakeVPA(t *testing.T) { v1.ResourceCPU, v1.ResourceMemory, }, ControlledValues: &controlledValues, + Mode: &containerMode, }, }, }, @@ -121,6 +132,9 @@ func TestMakeVPA(t *testing.T) { args: args{ objectMeta: &metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "function", + }, }, targetRef: &autov2.CrossVersionObjectReference{ APIVersion: "apps/v1", @@ -136,6 +150,9 @@ func TestMakeVPA(t *testing.T) { want: &vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "function", + }, }, TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling.k8s.io/v1", @@ -150,7 +167,14 @@ func TestMakeVPA(t *testing.T) { UpdatePolicy: &vpav1.PodUpdatePolicy{ UpdateMode: &mode, }, - ResourcePolicy: nil, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "pulsar-function", + Mode: &containerMode, + }, + }, + }, }, }, }, @@ -159,6 +183,9 @@ func TestMakeVPA(t *testing.T) { args: args{ objectMeta: &metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "sink", + }, }, targetRef: &autov2.CrossVersionObjectReference{ APIVersion: "apps/v1", @@ -168,7 +195,7 @@ func TestMakeVPA(t *testing.T) { vpa: &v1alpha1.VPASpec{ ResourcePolicy: &vpav1.PodResourcePolicy{ ContainerPolicies: []vpav1.ContainerResourcePolicy{{ - ContainerName: "test-container", + ContainerName: "pulsar-sink", MinAllowed: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), v1.ResourceMemory: resource.MustParse("100Mi"), @@ -189,6 +216,9 @@ func TestMakeVPA(t *testing.T) { want: &vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "sink", + }, }, TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling.k8s.io/v1", @@ -204,7 +234,7 @@ func TestMakeVPA(t *testing.T) { ResourcePolicy: &vpav1.PodResourcePolicy{ ContainerPolicies: []vpav1.ContainerResourcePolicy{ { - ContainerName: "test-container", + ContainerName: "pulsar-sink", MinAllowed: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), v1.ResourceMemory: resource.MustParse("100Mi"), @@ -217,6 +247,7 @@ func TestMakeVPA(t *testing.T) { v1.ResourceCPU, v1.ResourceMemory, }, ControlledValues: &controlledValues, + Mode: &containerMode, }, }, }, @@ -228,6 +259,9 @@ func TestMakeVPA(t *testing.T) { args: args{ objectMeta: &metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + }, }, targetRef: &autov2.CrossVersionObjectReference{ APIVersion: "apps/v1", @@ -239,6 +273,9 @@ func TestMakeVPA(t *testing.T) { want: &vpav1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + }, }, TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling.k8s.io/v1", @@ -250,8 +287,158 @@ func TestMakeVPA(t *testing.T) { Kind: "Deployment", Name: "test-deployment", }, - UpdatePolicy: nil, - ResourcePolicy: nil, + UpdatePolicy: nil, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "pulsar-source", + Mode: &containerMode, + }, + }, + }, + }, + }, + }, + { + name: "Generate VPA with resource unit successfully", + args: args{ + objectMeta: &metav1.ObjectMeta{ + Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + }, + }, + targetRef: &autov2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + }, + vpa: &v1alpha1.VPASpec{ + ResourceUnit: &v1alpha1.ResourceUnit{ + CPU: resource.MustParse("200m"), + Memory: resource.MustParse("800Mi"), + }, + }, + }, + want: &vpav1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + LabelCustomResourceUnit: "true", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling.k8s.io/v1", + Kind: "VerticalPodAutoscaler", + }, + Spec: vpav1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + }, + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: &off, + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "pulsar-source", + Mode: &containerMode, + }, + }, + }, + }, + }, + }, + { + name: "Generate VPA with resource unit and resource policy successfully", + args: args{ + objectMeta: &metav1.ObjectMeta{ + Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + }, + }, + targetRef: &autov2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + }, + vpa: &v1alpha1.VPASpec{ + ResourceUnit: &v1alpha1.ResourceUnit{ + CPU: resource.MustParse("200m"), + Memory: resource.MustParse("800Mi"), + }, + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: &mode, // should be set to 'Off' + MinReplicas: &minReplicas, + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "pulsar-source", + MinAllowed: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + MaxAllowed: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("1000Mi"), + }, + Mode: &containerOffMode, // should be converted to 'Auto' + ControlledValues: &controlledValues, + ControlledResources: &[]v1.ResourceName{v1.ResourceCPU}, + }, + { + ContainerName: "istio", // should be ignored + Mode: &containerMode, + }, + }, + }, + }, + }, + want: &vpav1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vpa", + Labels: map[string]string{ + LabelComponent: "source", + LabelCustomResourceUnit: "true", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling.k8s.io/v1", + Kind: "VerticalPodAutoscaler", + }, + Spec: vpav1.VerticalPodAutoscalerSpec{ + TargetRef: &autoscaling.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + }, + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: &off, + MinReplicas: &minReplicas, + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "pulsar-source", + Mode: &containerMode, + MinAllowed: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + MaxAllowed: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("1000Mi"), + }, + ControlledValues: &controlledValues, + ControlledResources: &[]v1.ResourceName{v1.ResourceCPU}, + }, + }, + }, }, }, },