Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replicas on same node #149

Closed
wants to merge 3 commits into from
Closed

Replicas on same node #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
91f024f
Add check for all replicas on same node
marians Jan 11, 2021
8e7a3ff
Boyscouting: add missing code 507
marians Jan 11, 2021
ab2f000
Add check for StatefulSets
marians Feb 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions assets/codes.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ codes:
506:
message: "At current load, Memory over allocated. Current:%s vs Requested:%s (%s)"
severity: 2
507:
message: Deployment references ServiceAccount %q which does not exist
severity: 3
508:
message: All %d replicas on the same node. Consider inter-pod anti-affinity
severity: 2
# HPA
600:
message: HPA %s references a Deployment %s which does not exist
Expand Down
3 changes: 3 additions & 0 deletions internal/issues/codes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ codes:
507:
message: Deployment references ServiceAccount %q which does not exist
severity: 3
508:
message: All %d replicas on the same node. Consider inter-pod anti-affinity
severity: 2

# -------------------------------------------------------------------------
# HPA
Expand Down
21 changes: 21 additions & 0 deletions internal/sanitize/dp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func (d *Deployment) Sanitize(ctx context.Context) error {

d.checkDeprecation(ctx, dp)
d.checkDeployment(ctx, dp)
d.checkSameNode(ctx, dp)
d.checkContainers(ctx, dp.Spec.Template.Spec)
pmx := client.PodsMetrics{}
podsMetrics(d, pmx)
Expand Down Expand Up @@ -105,6 +106,26 @@ func (d *Deployment) checkDeployment(ctx context.Context, dp *appsv1.Deployment)
}
}

// checkSameNode verifies if all replicas of the deployment are running on
// the same node.
func (d *Deployment) checkSameNode(ctx context.Context, dp *appsv1.Deployment) {
if *dp.Spec.Replicas <= 1 {
return
}

nodeMap := make(map[string]int)
for _, pod := range d.ListPodsBySelector(dp.Namespace, dp.Spec.Selector) {
if _, exists := nodeMap[pod.Spec.NodeName]; !exists {
nodeMap[pod.Spec.NodeName] = 0
}
nodeMap[pod.Spec.NodeName]++
}

if len(nodeMap) < 2 {
d.AddCode(ctx, 508, *dp.Spec.Replicas)
}
}

// CheckContainers runs thru deployment template and checks pod configuration.
func (d *Deployment) checkContainers(ctx context.Context, spec v1.PodSpec) {
c := NewContainer(internal.MustExtractFQN(ctx), d)
Expand Down
21 changes: 21 additions & 0 deletions internal/sanitize/sts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ func (s *StatefulSet) Sanitize(ctx context.Context) error {

s.checkDeprecation(ctx, st)
s.checkStatefulSet(ctx, st)
s.checkSameNode(ctx, st)
s.checkContainers(ctx, st)
s.checkUtilization(ctx, over, st, pmx)

Expand Down Expand Up @@ -120,6 +121,26 @@ func (s *StatefulSet) checkContainers(ctx context.Context, st *appsv1.StatefulSe
}
}

// checkSameNode verifies if all replicas of the StatefulSet are running on
// the same node.
func (s *StatefulSet) checkSameNode(ctx context.Context, st *appsv1.StatefulSet) {
if *st.Spec.Replicas <= 1 {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marians Check nil?

return
}

nodeMap := make(map[string]int)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't think we need to count here ie if the node is already in the map the pods are collocated

s.ListPodsBySelector(st.Namespace, st.Spec.Selector
make(map[string]struct{}, len(oo))
if _, ok := nodeMap[pod.Spec.NodeName]; ok {
   s.AddCode(...)
   return

That said I don't think these are valid general checks ie ES multiple shards can land on the same node? Thus that would raise a false positive??

for _, pod := range s.ListPodsBySelector(st.Namespace, st.Spec.Selector) {
if _, exists := nodeMap[pod.Spec.NodeName]; !exists {
nodeMap[pod.Spec.NodeName] = 0
}
nodeMap[pod.Spec.NodeName]++
}

if len(nodeMap) < 2 {
s.AddCode(ctx, 508, *st.Spec.Replicas)
}
}

func checkCPU(ctx context.Context, c CollectorLimiter, over bool, mx ConsumptionMetrics) {
cpuPerc := mx.ReqCPURatio()
if cpuPerc > 1 && cpuPerc > float64(c.CPUResourceLimits().UnderPerc) {
Expand Down