feat(inhibit): add inhibition metrics

Add metrics for inhibition rules:
- alertmanager_inhibitor_source_alerts_cache_size
- alertmanager_inhibitor_source_alerts_index_size
- alertmanager_inhibitor_rule_matches_duration_seconds
- alertmanager_inhibitor_rule_mutes_duration_seconds

Named rules get dedicated metric dimensions.
All rules are counted agains global metrics (rule="all").
If no rule mutes an alert the rule="none" dimension is used.

Other changes:
- Add Len() method to store.Alerts struct
- Add Len() method to inhibit.index struct

Signed-off-by: Siavash Safi <[email protected]>

Author: siavashs

cmd/alertmanager/main.go

@@ -104,7 +104,9 @@ var (
 		prometheus.GaugeOpts{
 			Name: "alertmanager_inhibition_rules",
 			Help: "Number of configured inhibition rules.",
-		})
+		},
+	)
+
 	promslogConfig = promslog.Config{}
 )
 
@@ -408,6 +410,7 @@ func run() int {
 	)
 
 	dispMetrics := dispatch.NewDispatcherMetrics(false, prometheus.DefaultRegisterer)
+	inhibitMetrics := inhibit.NewInhibitorMetrics(prometheus.DefaultRegisterer)
 	pipelineBuilder := notify.NewPipelineBuilder(prometheus.DefaultRegisterer, ff)
 	configLogger := logger.With("component", "configuration")
 	configCoordinator := config.NewCoordinator(
@@ -462,7 +465,7 @@ func run() int {
 		inhibitor.Stop()
 		disp.Stop()
 
-		inhibitor = inhibit.NewInhibitor(alerts, conf.InhibitRules, marker, logger)
+		inhibitor = inhibit.NewInhibitor(alerts, conf.InhibitRules, marker, logger, inhibitMetrics)
 		silencer := silence.NewSilencer(silences, marker, logger)
 
 		// An interface value that holds a nil concrete value is non-nil.

inhibit/index.go

@@ -55,3 +55,10 @@ func (c *index) Delete(key model.Fingerprint) {
 
 	delete(c.items, key)
 }
+
+func (c *index) Len() int {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+
+	return len(c.items)
+}

inhibit/inhibit.go

@@ -33,24 +33,27 @@ import (
 // currently active alerts and a set of inhibition rules. It implements the
 // Muter interface.
 type Inhibitor struct {
-	alerts provider.Alerts
-	rules  []*InhibitRule
-	marker types.AlertMarker
-	logger *slog.Logger
+	alerts  provider.Alerts
+	rules   []*InhibitRule
+	marker  types.AlertMarker
+	logger  *slog.Logger
+	metrics *InhibitorMetrics
 
 	mtx    sync.RWMutex
 	cancel func()
 }
 
 // NewInhibitor returns a new Inhibitor.
-func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger *slog.Logger) *Inhibitor {
+func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger *slog.Logger, metrics *InhibitorMetrics) *Inhibitor {
 	ih := &Inhibitor{
-		alerts: ap,
-		marker: mk,
-		logger: logger,
+		alerts:  ap,
+		marker:  mk,
+		logger:  logger,
+		metrics: metrics,
 	}
+
 	for _, cr := range rs {
-		r := NewInhibitRule(cr)
+		r := NewInhibitRule(cr, NewRuleMetrics(cr.Name, metrics))
 		ih.rules = append(ih.rules, r)
 	}
 	return ih
@@ -70,16 +73,22 @@ func (ih *Inhibitor) run(ctx context.Context) {
 				continue
 			}
 			// Update the inhibition rules' cache.
+			cached := 0
+			indexed := 0
 			for _, r := range ih.rules {
 				if r.SourceMatchers.Matches(a.Labels) {
 					if err := r.scache.Set(a); err != nil {
 						ih.logger.Error("error on set alert", "err", err)
 						continue
 					}
-
 					r.updateIndex(a)
+					cached += r.scache.Len()
+					indexed += r.sindex.Len()
+					r.metrics.ObserveSourceAlertsCacheSize(cached)
 				}
 			}
+			ih.metrics.ObserveSourceAlertsIndexSize(indexed)
+			ih.metrics.ObserveSourceAlertsCacheSize(cached)
 		}
 	}
 }
@@ -128,21 +137,28 @@ func (ih *Inhibitor) Stop() {
 // Mutes returns true iff the given label set is muted. It implements the Muter
 // interface.
 func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
+	start := time.Now()
 	fp := lset.Fingerprint()
 
 	for _, r := range ih.rules {
+		ruleStart := time.Now()
 		if !r.TargetMatchers.Matches(lset) {
 			// If target side of rule doesn't match, we don't need to look any further.
+			r.metrics.ObserveRuleMatchesDuration("false", time.Since(ruleStart))
 			continue
 		}
+		r.metrics.ObserveRuleMatchesDuration("true", time.Since(ruleStart))
 		// If we are here, the target side matches. If the source side matches, too, we
 		// need to exclude inhibiting alerts for which the same is true.
 		if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset)); eq {
 			ih.marker.SetInhibited(fp, inhibitedByFP.String())
+			r.metrics.ObserveRuleMutesDuration("true", time.Since(ruleStart))
 			return true
 		}
+		r.metrics.ObserveRuleMutesDuration("false", time.Since(ruleStart))
 	}
 	ih.marker.SetInhibited(fp)
+	ih.metrics.ObserveRuleMutesDuration("none", "false", time.Since(start))
 
 	return false
 }
@@ -173,14 +189,17 @@ type InhibitRule struct {
 	// The index items might overwrite eachother if multiple source alerts have exact equal labels.
 	// Overwrites only happen if the new source alert has bigger EndsAt value.
 	sindex *index
+
+	metrics *RuleMetrics
 }
 
 // NewInhibitRule returns a new InhibitRule based on a configuration definition.
-func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
+func NewInhibitRule(cr config.InhibitRule, metrics *RuleMetrics) *InhibitRule {
 	var (
 		sourcem labels.Matchers
 		targetm labels.Matchers
 	)
+
 	// cr.SourceMatch will be deprecated. This for loop appends regex matchers.
 	for ln, lv := range cr.SourceMatch {
 		matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
@@ -235,6 +254,7 @@ func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
 		Equal:          equal,
 		scache:         store.NewAlerts(),
 		sindex:         newIndex(),
+		metrics:        metrics,
 	}
 
 	rule.scache.SetGCCallback(rule.gcCallback)
@@ -262,6 +282,7 @@ func (r *InhibitRule) updateIndex(alert *types.Alert) {
 	if !ok {
 		// If not, add it.
 		r.sindex.Set(eq, fp)
+		r.metrics.ObserveSourceAlertsIndexSize(r.sindex.Len())
 		return
 	}
 	// If the indexed fingerprint is the same as the new fingerprint, do nothing.
@@ -274,12 +295,14 @@ func (r *InhibitRule) updateIndex(alert *types.Alert) {
 	if err != nil {
 		// failed to get the existing alert, overwrite the index.
 		r.sindex.Set(eq, fp)
+		r.metrics.ObserveSourceAlertsIndexSize(r.sindex.Len())
 		return
 	}
 
 	// If the new alert resolves after the existing alert, replace the index.
 	if existing.ResolvedAt(alert.EndsAt) {
 		r.sindex.Set(eq, fp)
+		r.metrics.ObserveSourceAlertsIndexSize(r.sindex.Len())
 		return
 	}
 	// If the existing alert resolves after the new alert, do nothing.
@@ -310,6 +333,8 @@ func (r *InhibitRule) gcCallback(alerts []types.Alert) {
 		fp := r.fingerprintEquals(a.Labels)
 		r.sindex.Delete(fp)
 	}
+	r.metrics.ObserveSourceAlertsCacheSize(r.scache.Len())
+	r.metrics.ObserveSourceAlertsIndexSize(r.sindex.Len())
 }
 
 // hasEqual checks whether the source cache contains alerts matching the equal

inhibit/inhibit_bench_test.go

@@ -198,7 +198,7 @@ func benchmarkMutes(b *testing.B, opts benchmarkOptions) {
 		}
 	}
 
-	ih := NewInhibitor(s, rules, m, promslog.NewNopLogger())
+	ih := NewInhibitor(s, rules, m, promslog.NewNopLogger(), NewInhibitorMetrics(r))
 	defer ih.Stop()
 	go ih.Run()
 

inhibit/inhibit_test.go

@@ -125,9 +125,10 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			r := &InhibitRule{
-				Equal:  map[model.LabelName]struct{}{},
-				scache: store.NewAlerts(),
-				sindex: newIndex(),
+				Equal:   map[model.LabelName]struct{}{},
+				scache:  store.NewAlerts(),
+				sindex:  newIndex(),
+				metrics: NewRuleMetrics("test", NewInhibitorMetrics(prometheus.NewRegistry())),
 			}
 			for _, ln := range c.equal {
 				r.Equal[ln] = struct{}{}
@@ -159,7 +160,7 @@ func TestInhibitRuleMatches(t *testing.T) {
 	}
 
 	m := types.NewMarker(prometheus.NewRegistry())
-	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger)
+	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 	now := time.Now()
 	// Active alert that matches the source filter of rule1.
 	sourceAlert1 := &types.Alert{
@@ -260,7 +261,7 @@ func TestInhibitRuleMatchers(t *testing.T) {
 	}
 
 	m := types.NewMarker(prometheus.NewRegistry())
-	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger)
+	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 	now := time.Now()
 	// Active alert that matches the source filter of rule1.
 	sourceAlert1 := &types.Alert{
@@ -369,8 +370,8 @@ func TestInhibitRuleName(t *testing.T) {
 		Equal: []string{"instance"},
 	}
 
-	rule1 := NewInhibitRule(config1)
-	rule2 := NewInhibitRule(config2)
+	rule1 := NewInhibitRule(config1, nil)
+	rule2 := NewInhibitRule(config2, nil)
 
 	require.Equal(t, "test-rule", rule1.Name, "Expected named rule to have adopt name from config")
 	require.Empty(t, rule2.Name, "Expected unnamed rule to have empty name")
@@ -498,7 +499,7 @@ func TestInhibit(t *testing.T) {
 	} {
 		ap := newFakeAlerts(tc.alerts)
 		mk := types.NewMarker(prometheus.NewRegistry())
-		inhibitor := NewInhibitor(ap, []config.InhibitRule{inhibitRule()}, mk, nopLogger)
+		inhibitor := NewInhibitor(ap, []config.InhibitRule{inhibitRule()}, mk, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 
 		go func() {
 			for ap.finished != nil {

inhibit/metric.go

@@ -0,0 +1,119 @@
+package inhibit
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// InhibitorMetrics represents metrics associated to an inhibitor.
+type InhibitorMetrics struct {
+	sourceAlertsCacheSize *prometheus.GaugeVec
+	sourceAlertsIndexSize *prometheus.GaugeVec
+	ruleMatchesDuration   *prometheus.SummaryVec
+	ruleMuteDuration      *prometheus.SummaryVec
+}
+
+// NewInhibitorMetrics returns a new InhibitorMetrics.
+func NewInhibitorMetrics(reg prometheus.Registerer) *InhibitorMetrics {
+	metrics := &InhibitorMetrics{
+		sourceAlertsCacheSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "alertmanager_inhibitor_source_alerts_cache_size",
+				Help: "Size of the source alerts cache in inhibition rules.",
+			},
+			[]string{"rule"},
+		),
+		sourceAlertsIndexSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "alertmanager_inhibitor_source_alerts_index_size",
+				Help: "Size of the source alerts index in inhibition rules.",
+			},
+			[]string{"rule"},
+		),
+		ruleMatchesDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name: "alertmanager_inhibitor_rule_matches_duration_seconds",
+				Help: "Summary of latencies for the matching of alerts by inhibition rules.",
+			},
+			[]string{"rule", "matched"},
+		),
+		ruleMuteDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name: "alertmanager_inhibitor_rule_mutes_duration_seconds",
+				Help: "Summary of latencies for the muting of alerts by inhibition rules.",
+			},
+			[]string{"rule", "muted"},
+		),
+	}
+	if reg != nil {
+		reg.MustRegister(
+			metrics.sourceAlertsCacheSize,
+			metrics.sourceAlertsIndexSize,
+			metrics.ruleMatchesDuration,
+			metrics.ruleMuteDuration,
+		)
+	}
+
+	metrics.ObserveSourceAlertsCacheSize(0)
+	metrics.ObserveSourceAlertsIndexSize(0)
+
+	return metrics
+}
+
+func (m *InhibitorMetrics) ObserveSourceAlertsCacheSize(size int) {
+	m.sourceAlertsCacheSize.With(prometheus.Labels{"rule": "all"}).Set(float64(size))
+}
+
+func (m *InhibitorMetrics) ObserveSourceAlertsIndexSize(size int) {
+	m.sourceAlertsIndexSize.With(prometheus.Labels{"rule": "all"}).Set(float64(size))
+}
+
+func (m *InhibitorMetrics) ObserveRuleMutesDuration(rule, muted string, duration time.Duration) {
+	m.ruleMuteDuration.With(prometheus.Labels{"rule": rule, "muted": muted}).Observe(duration.Seconds())
+}
+
+type RuleMetrics struct {
+	ruleName              string
+	matchesDuration       *prometheus.SummaryVec
+	mutesDuration         *prometheus.SummaryVec
+	sourceAlertsCacheSize *prometheus.GaugeVec
+	sourceAlertsIndexSize *prometheus.GaugeVec
+}
+
+func NewRuleMetrics(name string, metrics *InhibitorMetrics) *RuleMetrics {
+	rm := &RuleMetrics{
+		ruleName:              name,
+		matchesDuration:       metrics.ruleMatchesDuration,
+		mutesDuration:         metrics.ruleMuteDuration,
+		sourceAlertsCacheSize: metrics.sourceAlertsCacheSize,
+		sourceAlertsIndexSize: metrics.sourceAlertsIndexSize,
+	}
+	rm.ObserveSourceAlertsCacheSize(0)
+	rm.ObserveSourceAlertsIndexSize(0)
+	return rm
+}
+
+func (rm *RuleMetrics) ObserveSourceAlertsCacheSize(size int) {
+	// If rule name is empty, skip per rule gauge metrics
+	if rm.ruleName == "" {
+		return
+	}
+	rm.sourceAlertsCacheSize.With(prometheus.Labels{"rule": rm.ruleName}).Set(float64(size))
+}
+
+func (rm *RuleMetrics) ObserveSourceAlertsIndexSize(size int) {
+	// If rule name is empty, skip per rule gauge metrics
+	if rm.ruleName == "" {
+		return
+	}
+	rm.sourceAlertsIndexSize.With(prometheus.Labels{"rule": rm.ruleName}).Set(float64(size))
+}
+
+func (rm *RuleMetrics) ObserveRuleMatchesDuration(matched string, duration time.Duration) {
+	rm.matchesDuration.With(prometheus.Labels{"rule": rm.ruleName, "matched": matched}).Observe(duration.Seconds())
+}
+
+func (rm *RuleMetrics) ObserveRuleMutesDuration(muted string, duration time.Duration) {
+	rm.mutesDuration.With(prometheus.Labels{"rule": rm.ruleName, "muted": muted}).Observe(duration.Seconds())
+}