Skip to content

fix: network-partion targets not found and chaos injection failure #758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: network-partion targets not found and chaos injection failure #758

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions chaoslib/litmus/pod-network-partition/lib/network-policy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ package lib

import (
"fmt"
"github.com/litmuschaos/litmus-go/pkg/cerrors"
"github.com/litmuschaos/litmus-go/pkg/clients"
"github.com/palantir/stacktrace"
"strings"

network_chaos "github.com/litmuschaos/litmus-go/chaoslib/litmus/network-chaos/lib"
"github.com/litmuschaos/litmus-go/pkg/cerrors"
"github.com/litmuschaos/litmus-go/pkg/clients"
experimentTypes "github.com/litmuschaos/litmus-go/pkg/generic/pod-network-partition/types"
"github.com/palantir/stacktrace"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
networkv1 "k8s.io/api/networking/v1"
Expand Down Expand Up @@ -53,7 +53,7 @@ func (np *NetworkPolicy) getNetworkPolicyDetails(experimentsDetails *experimentT
setNamespaceSelector(experimentsDetails.NamespaceSelector)

// sets the ports for the traffic control
if err := np.setPort(experimentsDetails.PORTS); err != nil {
if err := np.setPort(experimentsDetails.Ports); err != nil {
return stacktrace.Propagate(err, "could not set port")
}

Expand Down
44 changes: 12 additions & 32 deletions chaoslib/litmus/pod-network-partition/lib/pod-network-partition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/litmuschaos/litmus-go/pkg/utils/retry"
"github.com/litmuschaos/litmus-go/pkg/utils/stringutils"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
networkv1 "k8s.io/api/networking/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand All @@ -48,23 +47,11 @@ func PrepareAndInjectChaos(ctx context.Context, experimentsDetails *experimentTy
// Catch and relay certain signal(s) to abort channel.
signal.Notify(abort, os.Interrupt, syscall.SIGTERM)

// validate the appLabels
if chaosDetails.AppDetail == nil {
return cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Reason: "provide the appLabel"}
// Network policy will be created based on POD_SELECTOR and NAMESPACE_SELECTOR
if experimentsDetails.PodSelector == "" && experimentsDetails.NamespaceSelector == "" {
return cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Reason: "provide POD_SELECTOR or NAMESPACE_SELECTOR for network policy"}
}

// Get the target pod details for the chaos execution
targetPodList, err := common.GetPodList("", 100, clients, chaosDetails)
if err != nil {
return stacktrace.Propagate(err, "could not get target pods")
}

podNames := []string{}
for _, pod := range targetPodList.Items {
podNames = append(podNames, pod.Name)
}
log.Infof("Target pods list for chaos, %v", podNames)

// generate a unique string
runID := stringutils.GetRunID()

Expand All @@ -91,7 +78,7 @@ func PrepareAndInjectChaos(ctx context.Context, experimentsDetails *experimentTy
})

// watching for the abort signal and revert the chaos
go abortWatcher(experimentsDetails, clients, chaosDetails, resultDetails, &targetPodList, runID)
go abortWatcher(experimentsDetails, clients, chaosDetails, resultDetails, runID)

// run the probes during chaos
if len(resultDetails.ProbeDetails) != 0 {
Expand All @@ -109,10 +96,8 @@ func PrepareAndInjectChaos(ctx context.Context, experimentsDetails *experimentTy
if err := createNetworkPolicy(ctx, experimentsDetails, clients, np, runID); err != nil {
return stacktrace.Propagate(err, "could not create network policy")
}
// updating chaos status to injected for the target pods
for _, pod := range targetPodList.Items {
common.SetTargets(pod.Name, "injected", "pod", chaosDetails)
}
// Set chaos status to injected
common.SetTargets("network-policy", "injected", "pod", chaosDetails)
}

// verify the presence of network policy inside cluster
Expand All @@ -124,14 +109,12 @@ func PrepareAndInjectChaos(ctx context.Context, experimentsDetails *experimentTy
common.WaitForDuration(experimentsDetails.ChaosDuration)

// deleting the network policy after chaos duration over
if err := deleteNetworkPolicy(experimentsDetails, clients, &targetPodList, chaosDetails, experimentsDetails.Timeout, experimentsDetails.Delay, runID); err != nil {
if err := deleteNetworkPolicy(experimentsDetails, clients, experimentsDetails.Timeout, experimentsDetails.Delay, runID); err != nil {
return stacktrace.Propagate(err, "could not delete network policy")
}

// updating chaos status to reverted for the target pods
for _, pod := range targetPodList.Items {
common.SetTargets(pod.Name, "reverted", "pod", chaosDetails)
}
// Set chaos status to reverted
common.SetTargets("network-policy", "reverted", "pod", chaosDetails)

//Waiting for the ramp time after chaos injection
if experimentsDetails.RampTime != 0 {
Expand Down Expand Up @@ -176,7 +159,7 @@ func createNetworkPolicy(ctx context.Context, experimentsDetails *experimentType
}

// deleteNetworkPolicy deletes the network policy and wait until the network policy deleted completely
func deleteNetworkPolicy(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, targetPodList *corev1.PodList, chaosDetails *types.ChaosDetails, timeout, delay int, runID string) error {
func deleteNetworkPolicy(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, timeout, delay int, runID string) error {
name := experimentsDetails.ExperimentName + "-np-" + runID
labels := "name=" + experimentsDetails.ExperimentName + "-np-" + runID
if err := clients.KubeClient.NetworkingV1().NetworkPolicies(experimentsDetails.AppNS).Delete(context.Background(), name, v1.DeleteOptions{}); err != nil {
Expand All @@ -200,9 +183,6 @@ func deleteNetworkPolicy(experimentsDetails *experimentTypes.ExperimentDetails,
return err
}

for _, pod := range targetPodList.Items {
common.SetTargets(pod.Name, "reverted", "pod", chaosDetails)
}
return nil
}

Expand All @@ -225,7 +205,7 @@ func checkExistenceOfPolicy(experimentsDetails *experimentTypes.ExperimentDetail
}

// abortWatcher continuously watch for the abort signals
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails, targetPodList *corev1.PodList, runID string) {
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails, runID string) {
// waiting till the abort signal received
<-abort

Expand All @@ -245,7 +225,7 @@ func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients
continue
}

if err := deleteNetworkPolicy(experimentsDetails, clients, targetPodList, chaosDetails, 2, 1, runID); err != nil {
if err := deleteNetworkPolicy(experimentsDetails, clients, 2, 1, runID); err != nil {
log.Errorf("unable to delete network policy, err: %v", err)
}
retry--
Expand Down
20 changes: 4 additions & 16 deletions pkg/generic/pod-network-partition/environment/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ChaosDuration, _ = strconv.Atoi(types.Getenv("TOTAL_CHAOS_DURATION", "30"))
experimentDetails.RampTime, _ = strconv.Atoi(types.Getenv("RAMP_TIME", "0"))
experimentDetails.AppNS = types.Getenv("APP_NAMESPACE", "")
experimentDetails.AppLabel = types.Getenv("APP_LABEL", "")
experimentDetails.AppKind = types.Getenv("APP_KIND", "")
experimentDetails.AppLabel = types.Getenv("APP_LABEL", "") // Target pod labels on which network partition will be applied
experimentDetails.ChaosUID = clientTypes.UID(types.Getenv("CHAOS_UID", ""))
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
experimentDetails.ChaosPodName = types.Getenv("POD_NAME", "")
Expand All @@ -29,18 +28,7 @@ func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.DestinationHosts = types.Getenv("DESTINATION_HOSTS", "")
experimentDetails.LIBImagePullPolicy = types.Getenv("LIB_IMAGE_PULL_POLICY", "Always")
experimentDetails.PolicyTypes = types.Getenv("POLICY_TYPES", "all")
experimentDetails.PodSelector = types.Getenv("POD_SELECTOR", "")
experimentDetails.NamespaceSelector = types.Getenv("NAMESPACE_SELECTOR", "")
experimentDetails.PORTS = types.Getenv("PORTS", "")

experimentDetails.AppNS, experimentDetails.AppKind, experimentDetails.AppLabel = getAppDetails()
}

func getAppDetails() (string, string, string) {
targets := types.Getenv("TARGETS", "")
app := types.GetTargets(targets)
if len(app) != 0 {
return app[0].Namespace, app[0].Kind, app[0].Labels[0]
}
return "", "", ""
experimentDetails.PodSelector = types.Getenv("POD_SELECTOR", "") // Pod selector to filter pods for network partition ingress or egress rules
experimentDetails.NamespaceSelector = types.Getenv("NAMESPACE_SELECTOR", "") // Namespace selector to filter namespaces for network partition ingress or egress rules
experimentDetails.Ports = types.Getenv("PORTS", "") // Ports to be used for network partition
}
3 changes: 1 addition & 2 deletions pkg/generic/pod-network-partition/types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ type ExperimentDetails struct {
RampTime int
AppNS string
AppLabel string
AppKind string
ChaosUID clientTypes.UID
InstanceID string
LIBImagePullPolicy string
Expand All @@ -26,5 +25,5 @@ type ExperimentDetails struct {
PolicyTypes string
PodSelector string
NamespaceSelector string
PORTS string
Ports string
}
30 changes: 22 additions & 8 deletions pkg/utils/common/pods.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ func GetHelperLabels(labels map[string]string, runID, experimentName string) map
return labels
}

// VerifyExistanceOfPods check the availibility of list of pods
func VerifyExistanceOfPods(namespace, pods string, clients clients.ClientSets) (bool, error) {
// VerifyExistenceOfPods check the availability of list of pods
func VerifyExistenceOfPods(namespace, pods string, clients clients.ClientSets) (bool, error) {

if strings.TrimSpace(pods) == "" {
return false, nil
Expand Down Expand Up @@ -150,7 +150,7 @@ func GetPodList(targetPods string, podAffPerc int, clients clients.ClientSets, c
namespace = chaosDetails.ChaosNamespace
}

isPodsAvailable, err := VerifyExistanceOfPods(namespace, targetPods, clients)
isPodsAvailable, err := VerifyExistenceOfPods(namespace, targetPods, clients)
if err != nil {
return core_v1.PodList{}, stacktrace.Propagate(err, "could not verify existence of TARGET_PODS")
}
Expand Down Expand Up @@ -274,12 +274,26 @@ func GetTargetPodsWhenTargetPodsENVNotSet(podAffPerc int, clients clients.Client
for _, target := range chaosDetails.AppDetail {
switch target.Kind {
case "pod":
for _, name := range target.Names {
pod, err := clients.KubeClient.CoreV1().Pods(target.Namespace).Get(context.Background(), name, v1.GetOptions{})
if err != nil {
return finalPods, cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Target: fmt.Sprintf("{podName: %s, namespace: %s}", name, target.Namespace), Reason: err.Error()}
if target.Names != nil {
for _, name := range target.Names {
pod, err := clients.KubeClient.CoreV1().Pods(target.Namespace).Get(context.Background(), name, v1.GetOptions{})
if err != nil {
return finalPods, cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Target: fmt.Sprintf("{podName: %s, namespace: %s}", name, target.Namespace), Reason: err.Error()}
}
finalPods.Items = append(finalPods.Items, *pod)
}
} else {
for _, label := range target.Labels {
pods, err := clients.KubeClient.CoreV1().Pods(target.Namespace).List(context.Background(), v1.ListOptions{LabelSelector: label})
if err != nil {
return finalPods, cerrors.Error{ErrorCode: cerrors.ErrorTypeTargetSelection, Target: fmt.Sprintf("{podLabel: %s, namespace: %s}", label, target.Namespace), Reason: err.Error()}
}
filteredPods, err := filterPodsByOwnerKind(pods.Items, target, clients)
if err != nil {
return finalPods, stacktrace.Propagate(err, "could not identify parent type from pod")
}
finalPods.Items = append(finalPods.Items, filteredPods...)
}
finalPods.Items = append(finalPods.Items, *pod)
}
podKind = true
default:
Expand Down