feat: add event handling strategy changes in syslog health monitor

data-models/protobufs/health_event.proto

@@ -29,6 +29,15 @@ message HealthEvents {
   repeated HealthEvent events = 2;
 }
 
+// ProcessingStrategy defines how downstream modules should handle the event.
+// EXECUTE_REMEDIATION: normal behavior; downstream modules may update cluster state.
+// STORE_ONLY: observability-only behavior; event should be persisted/exported but should not modify cluster resources.
+enum ProcessingStrategy {
+  UNSPECIFIED = 0;
+  EXECUTE_REMEDIATION = 1;
+  STORE_ONLY = 2;
+}
+
 enum RecommendedAction {
   NONE = 0;
   COMPONENT_RESET = 2;
@@ -66,6 +75,7 @@ message HealthEvent {
   string nodeName = 13;
   BehaviourOverrides quarantineOverrides = 14;
   BehaviourOverrides drainOverrides = 15;
+  ProcessingStrategy processingStrategy = 16;
 }
 
 message BehaviourOverrides {

distros/kubernetes/nvsentinel/charts/syslog-health-monitor/templates/_helpers.tpl

@@ -100,6 +100,8 @@ spec:
             - "{{ join "," $root.Values.enabledChecks }}"
             - "--metadata-path"
             - "{{ $root.Values.global.metadataPath }}"
+            - "--processing-strategy"
+            - {{ $root.Values.processingStrategy }}
           resources:
             {{- toYaml $root.Values.resources | nindent 12 }}
           ports:

distros/kubernetes/nvsentinel/charts/syslog-health-monitor/values.yaml

@@ -50,3 +50,10 @@ tolerations: []
 journalHostPath: /var/log
 
 logLevel: info
+
+# Processing strategy for health events
+# valid values: EXECUTE_REMEDIATION, STORE_ONLY
+# default: EXECUTE_REMEDIATION
+# EXECUTE_REMEDIATION: normal behavior; downstream modules may update cluster state.
+# STORE_ONLY: observability-only behavior; event should be persisted/exported but should not modify cluster resources (i.e., no node conditions, no quarantine, no drain, no remediation).
+processingStrategy: EXECUTE_REMEDIATION

event-exporter/pkg/transformer/cloudevents.go

@@ -63,6 +63,7 @@ func ToCloudEvent(event *pb.HealthEvent, metadata map[string]string) (*CloudEven
 		"entitiesImpacted":   entities,
 		"generatedTimestamp": timestamp,
 		"nodeName":           event.NodeName,
+		"processingStrategy": event.ProcessingStrategy.String(),
 	}
 
 	if len(event.Metadata) > 0 {

event-exporter/pkg/transformer/cloudevents_test.go

@@ -66,6 +66,7 @@ func TestToCloudEvent(t *testing.T) {
 					Force: false,
 					Skip:  true,
 				},
+				ProcessingStrategy: pb.ProcessingStrategy_STORE_ONLY,
 			},
 			metadata: map[string]string{
 				"cluster":     "prod-cluster-1",
@@ -102,6 +103,9 @@ func TestToCloudEvent(t *testing.T) {
 				if healthEvent["recommendedAction"] != "RESTART_VM" {
 					t.Errorf("recommendedAction = %v, want %v", healthEvent["recommendedAction"], "RESTART_VM")
 				}
+				if healthEvent["processingStrategy"] != "STORE_ONLY" {
+					t.Errorf("processingStrategy = %v, want STORE_ONLY", healthEvent["processingStrategy"])
+				}
 
 				entities := healthEvent["entitiesImpacted"].([]map[string]any)
 				if len(entities) != 2 {

fault-quarantine/pkg/evaluator/rule_evaluator_test.go

@@ -260,6 +260,7 @@ func TestRoundTrip(t *testing.T) {
 			"nanos":   float64(eventTime.GetNanos()),
 		},
 		"nodeName":            "test-node",
+		"processingStrategy":  float64(0),
 		"quarantineOverrides": nil,
 		"drainOverrides":      nil,
 	}

fault-quarantine/pkg/initializer/init.go

@@ -63,7 +63,7 @@ func InitializeAll(ctx context.Context, params InitializationParams) (*Component
 	}
 
 	builder := client.GetPipelineBuilder()
-	pipeline := builder.BuildAllHealthEventInsertsPipeline()
+	pipeline := builder.BuildProcessableHealthEventInsertsPipeline()
 
 	var tomlCfg config.TomlConfig
 	if err := configmanager.LoadTOMLConfig(params.TomlConfigPath, &tomlCfg); err != nil {

health-monitors/csp-health-monitor/pkg/triggerengine/trigger.go

@@ -359,6 +359,9 @@ func (e *Engine) mapMaintenanceEventToHealthEvent(
 		Metadata:           event.Metadata, // Pass along metadata
 		NodeName:           event.NodeName, // K8s node name
 		GeneratedTimestamp: timestamppb.New(time.Now()),
+		// TODO: Remove hardcoded processing strategy and make it configurable via the config file.
+		// PR: https://github.com/NVIDIA/NVSentinel/pull/641
+		ProcessingStrategy: pb.ProcessingStrategy_EXECUTE_REMEDIATION,
 	}
 
 	return healthEvent, nil

health-monitors/csp-health-monitor/pkg/triggerengine/trigger_test.go

@@ -240,8 +240,9 @@ func TestMapMaintenanceEventToHealthEvent(t *testing.T) {
 				EntitiesImpacted: []*pb.Entity{
 					{EntityType: "gce_instance", EntityValue: "instance-123"},
 				},
-				Metadata: map[string]string{"key": "value"},
-				NodeName: "node-a",
+				Metadata:           map[string]string{"key": "value"},
+				NodeName:           "node-a",
+				ProcessingStrategy: pb.ProcessingStrategy_EXECUTE_REMEDIATION,
 			},
 		},
 		{
@@ -268,7 +269,8 @@ func TestMapMaintenanceEventToHealthEvent(t *testing.T) {
 				EntitiesImpacted: []*pb.Entity{
 					{EntityType: "EC2", EntityValue: "i-abcdef"},
 				},
-				NodeName: "node-b",
+				NodeName:           "node-b",
+				ProcessingStrategy: pb.ProcessingStrategy_EXECUTE_REMEDIATION,
 			},
 		},
 		{
@@ -331,7 +333,8 @@ func TestMapMaintenanceEventToHealthEvent(t *testing.T) {
 				EntitiesImpacted: []*pb.Entity{
 					{EntityType: "gce_instance", EntityValue: "instance-789"},
 				},
-				NodeName: "node-e",
+				NodeName:           "node-e",
+				ProcessingStrategy: pb.ProcessingStrategy_EXECUTE_REMEDIATION,
 			},
 		},
 	}