Improvements: rename KvCacheTokensTotal, expand help text, use unit label

- Rename KvCacheTokensTotal -> KvCacheTokensCapacity on VariantDecision,
  Actuator.EmitSaturationMetrics, MetricsEmitter.EmitSaturationMetrics,
  DeleteSaturationMetrics, and the Prometheus metric name itself
  (wva_kv_cache_tokens_total -> wva_kv_cache_tokens_capacity). "Total" was
  confusing — the metric is a gauge of capacity, not a cumulative counter.

- Replace the analyzer_version="v1"/"v2" label on wva_required_capacity
  with a unit="binary"/"continuous" label. The label's purpose is to
  describe the unit of the metric value (a boolean scale-up signal in V1,
  a continuous token demand in V2), not the code path that produced it.
  "binary"/"continuous" remains meaningful after V1 is deprecated, whereas
  "v1"/"v2" becomes vestigial.

  Rename VariantDecision.AnalyzerVersion -> RequiredCapacityUnit.
  Rename constants.LabelAnalyzerVersion -> LabelUnit.
  Rename constants.AnalyzerVersionV1/V2 -> UnitBinary/UnitContinuous.

- Expand help strings on wva_saturation_utilization, wva_spare_capacity,
  wva_kv_cache_tokens_used, and wva_kv_cache_tokens_capacity to specify
  what is being measured (KV-cache) and how V1 vs V2 paths differ.

- Use constants.LabelUnit, UnitBinary, UnitContinuous in the
  wva_required_capacity help string via fmt.Sprintf, for consistency with
  how labels are referenced elsewhere.

Author: ev-shindin

internal/actuator/actuator.go

@@ -101,11 +101,19 @@ func (a *Actuator) EmitSaturationMetrics(ctx context.Context, decision interface
 		decision.VariantName,
 		decision.Namespace,
 		decision.AcceleratorName,
-		decision.AnalyzerVersion,
+		decision.RequiredCapacityUnit,
 		decision.Utilization,
 		decision.SpareCapacity,
 		decision.RequiredCapacity,
 		decision.KvCacheTokensUsed,
-		decision.KvCacheTokensTotal,
+		decision.KvCacheTokensCapacity,
 	)
 }
+
+// DeleteSaturationMetricsForVariant removes all saturation metric series for a
+// variant. Call this when the current optimization cycle produced no fresh
+// decision for the variant, or when the VA is being deleted — so dashboards
+// don't show stale values.
+func (a *Actuator) DeleteSaturationMetricsForVariant(variantName, namespace string) {
+	a.MetricsEmitter.DeleteSaturationMetricsForVariant(variantName, namespace)
+}

internal/constants/metrics.go

@@ -119,19 +119,19 @@ const (
 
 	// WVARequiredCapacity is a gauge that tracks model-level required capacity.
 	// >0 means scale-up needed.
-	// Units differ by analyzer (use the analyzer_version label to distinguish):
-	//   - V1: binary signal (0.0 = no scale-up, 1.0 = scale-up needed)
-	//   - V2: continuous token-based demand
-	// Labels: variant_name, namespace, analyzer_version
+	// Value semantics differ by analyzer (use the "unit" label to distinguish):
+	//   - unit="binary"     (V1): 0.0 = no scale-up, 1.0 = scale-up needed
+	//   - unit="continuous" (V2): continuous token-based demand
+	// Labels: variant_name, namespace, unit
 	WVARequiredCapacity = "wva_required_capacity"
 
 	// WVAKvCacheTokensUsed is a gauge that tracks total KV cache tokens currently in use per variant.
 	// Labels: variant_name, namespace
 	WVAKvCacheTokensUsed = "wva_kv_cache_tokens_used"
 
-	// WVAKvCacheTokensTotal is a gauge that tracks total KV cache token capacity per variant.
+	// WVAKvCacheTokensCapacity is a gauge that tracks total KV cache token capacity per variant.
 	// Labels: variant_name, namespace
-	WVAKvCacheTokensTotal = "wva_kv_cache_tokens_total"
+	WVAKvCacheTokensCapacity = "wva_kv_cache_tokens_capacity"
 )
 
 // Metric Label Names
@@ -144,11 +144,16 @@ const (
 	LabelReason             = "reason"
 	LabelAcceleratorType    = "accelerator_type"
 	LabelControllerInstance = "controller_instance"
-	LabelAnalyzerVersion    = "analyzer_version"
+	// LabelUnit distinguishes the unit of a metric value when a single metric name
+	// carries values with different semantic units. Currently applied to
+	// wva_required_capacity, whose value is either a binary scale-up signal (V1)
+	// or a continuous token-demand value (V2).
+	LabelUnit = "unit"
 )
 
-// Analyzer version label values used in saturation metrics.
+// Values for the LabelUnit Prometheus label, describing how to interpret the
+// metric value ("binary" 0/1 vs. "continuous" absolute quantity).
 const (
-	AnalyzerVersionV1 = "v1"
-	AnalyzerVersionV2 = "v2"
+	UnitBinary     = "binary"
+	UnitContinuous = "continuous"
 )

internal/engines/saturation/engine.go

@@ -764,17 +764,23 @@ func enrichDecisionsFromReplicaMetrics(decisions []interfaces.VariantDecision, r
 	for i := range decisions {
 		d := &decisions[i]
 		d.RequiredCapacity = requiredCapacity
-		d.AnalyzerVersion = constants.AnalyzerVersionV1
+		d.RequiredCapacityUnit = constants.UnitBinary
 		if a, ok := agg[d.VariantName]; ok && a.count > 0 {
 			d.KvCacheTokensUsed = a.kvUsed
-			d.KvCacheTokensTotal = a.kvTotal
+			d.KvCacheTokensCapacity = a.kvTotal
+			// V1 reasons about saturation per-replica using KvCacheUsage fractions
+			// (rm.KvCacheUsage is 0.0-1.0), not tokens. Report the mean of those
+			// per-replica fractions as the variant-level utilization — this
+			// matches what the V1 analyzer actually evaluates against its
+			// thresholds. V2 uses a different (token-demand / capacity) formula;
+			// see the field doc on VariantDecision.Utilization.
 			d.Utilization = a.kvUsageSum / float64(a.count)
 		}
 	}
 }
 
-// enrichDecisionsWithKvTokenData sets KvCacheTokensUsed, KvCacheTokensTotal, and
-// AnalyzerVersion on decisions from replica metrics aggregated per (model, variant).
+// enrichDecisionsWithKvTokenData sets KvCacheTokensUsed, KvCacheTokensCapacity, and
+// RequiredCapacityUnit on decisions from replica metrics aggregated per (model, variant).
 // Used by V2 path where Utilization and RequiredCapacity are already set from
 // AnalyzerResult.
 //
@@ -805,10 +811,10 @@ func enrichDecisionsWithKvTokenData(decisions []interfaces.VariantDecision, mode
 
 	for i := range decisions {
 		d := &decisions[i]
-		d.AnalyzerVersion = constants.AnalyzerVersionV2
+		d.RequiredCapacityUnit = constants.UnitContinuous
 		if a, ok := agg[variantKey{modelID: d.ModelID, variant: d.VariantName}]; ok {
 			d.KvCacheTokensUsed = a.kvUsed
-			d.KvCacheTokensTotal = a.kvTotal
+			d.KvCacheTokensCapacity = a.kvTotal
 		}
 	}
 }
@@ -1157,14 +1163,18 @@ func (e *Engine) applySaturationDecisions(
 		}
 
 		// Emit saturation and capacity metrics for observability.
-		// Note: stale time series for deleted VAs are not cleaned up automatically here.
-		// The metrics package exposes DeleteSaturationMetrics for callers (e.g., the
-		// VariantAutoscaling reconciler's delete handler / finalizer) to remove series
-		// when a VA is removed.
+		// When this cycle produced no fresh decision for the variant, actively
+		// clear the existing series so dashboards show a gap ("no fresh data")
+		// rather than stale values that would otherwise persist until Prometheus'
+		// 5-minute staleness marker fires. For fully-deleted VAs, additional
+		// cleanup via the reconciler's delete handler / finalizer is still
+		// required (see DeleteSaturationMetricsForVariant).
 		if hasDecision {
 			if err := act.EmitSaturationMetrics(ctx, decision); err != nil {
 				logger.Error(err, "Failed to emit saturation metrics", "variant", updateVa.Name)
 			}
+		} else {
+			act.DeleteSaturationMetricsForVariant(updateVa.Name, updateVa.Namespace)
 		}
 
 		// Update Shared State and Trigger Reconcile via Channel

internal/interfaces/saturation_analyzer.go

@@ -195,25 +195,32 @@ type VariantDecision struct {
 	// V1: threshold-relative spare KV capacity (AvgSpareKvCapacity).
 	// V2: 1.0 - Utilization (absolute spare).
 	SpareCapacity float64
-	// Utilization is the variant-level utilization ratio (0.0-1.0).
-	// V2: from AnalyzerResult.VariantCapacities[].Utilization.
-	// V1: average KvCacheUsage across this variant's replicas.
+	// Utilization is the variant-level utilization ratio (0.0-1.0) reported for
+	// observability. The exact formula differs by analyzer because V1 and V2
+	// reason about saturation differently:
+	//   V1: mean of per-replica KvCacheUsage fractions (matches what V1's
+	//       per-replica threshold check operates on).
+	//   V2: TotalDemand / TotalCapacity from AnalyzerResult (token-demand-based).
+	// For uniform-capacity replicas the two are numerically equivalent; for
+	// mixed-capacity replicas V2's value is capacity-weighted.
 	Utilization float64
 	// KvCacheTokensUsed is the sum of TokensInUse across this variant's replicas.
 	KvCacheTokensUsed int64
-	// KvCacheTokensTotal is the sum of TotalKvCapacityTokens across this variant's replicas.
-	KvCacheTokensTotal int64
+	// KvCacheTokensCapacity is the sum of TotalKvCapacityTokens across this variant's replicas.
+	KvCacheTokensCapacity int64
 	// RequiredCapacity is the model-level required capacity (>0 means scale-up needed).
 	// Same value for all variants of a model.
 	// V1: binary (1.0 if shouldScaleUp, else 0.0).
 	// V2: continuous token-based demand from AnalyzerResult.
-	// Use AnalyzerVersion to disambiguate the units when consuming this field
+	// Use RequiredCapacityUnit to disambiguate the units when consuming this field
 	// (or its corresponding Prometheus metric).
 	RequiredCapacity float64
-	// AnalyzerVersion identifies which analyzer produced this decision ("v1" or "v2").
-	// Exposed as a Prometheus label on saturation metrics so dashboards can filter
-	// by analyzer to handle the V1/V2 unit difference in RequiredCapacity.
-	AnalyzerVersion string
+	// RequiredCapacityUnit describes the unit of RequiredCapacity ("binary" or "continuous").
+	// Exposed as the `unit` Prometheus label on wva_required_capacity so dashboards
+	// can filter by semantics rather than by which analyzer produced the value.
+	//   "binary":     V1 path, value is 0.0 or 1.0
+	//   "continuous": V2 path, value is a token-demand magnitude
+	RequiredCapacityUnit string
 	// ScaleTargetRef references the Deployment/StatefulSet for scheduling constraints
 	ScaleTargetRef *autoscalingv2.CrossVersionObjectReference
 

internal/metrics/metrics.go

@@ -25,7 +25,7 @@ var (
 	spareCapacity         *prometheus.GaugeVec
 	requiredCapacity      *prometheus.GaugeVec
 	kvCacheTokensUsed     *prometheus.GaugeVec
-	kvCacheTokensTotal    *prometheus.GaugeVec
+	kvCacheTokensCapacity *prometheus.GaugeVec
 
 	// controllerInstance stores the optional controller instance identifier.
 	// When set, it's added as a label to all emitted metrics.
@@ -51,9 +51,9 @@ func InitMetrics(registry prometheus.Registerer) error {
 	scalingLabels := []string{constants.LabelVariantName, constants.LabelNamespace, constants.LabelDirection, constants.LabelReason}
 	// modelLabels: variant_name + namespace only (no accelerator_type) for model-level and token metrics
 	modelLabels := []string{constants.LabelVariantName, constants.LabelNamespace}
-	// requiredCapacityLabels: model labels + analyzer_version to disambiguate V1 (binary)
-	// vs V2 (continuous tokens) units of the wva_required_capacity gauge
-	requiredCapacityLabels := []string{constants.LabelVariantName, constants.LabelNamespace, constants.LabelAnalyzerVersion}
+	// requiredCapacityLabels: model labels + "unit" to disambiguate V1 (binary 0/1)
+	// vs V2 (continuous token demand) values of the wva_required_capacity gauge
+	requiredCapacityLabels := []string{constants.LabelVariantName, constants.LabelNamespace, constants.LabelUnit}
 
 	if controllerInstance != "" {
 		baseLabels = append(baseLabels, constants.LabelControllerInstance)
@@ -93,35 +93,35 @@ func InitMetrics(registry prometheus.Registerer) error {
 	saturationUtilization = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: constants.WVASaturationUtilization,
-			Help: "Per-variant utilization ratio (0.0-1.0) from saturation analysis",
+			Help: "Per-variant utilization ratio (0.0-1.0) from saturation analysis. V1 path: mean of per-replica KV-cache-usage fractions (matches the per-replica threshold V1 checks). V2 path: TotalDemand / TotalCapacity from the analyzer result. Numerically equivalent for uniform-capacity replicas; V2 is capacity-weighted for mixed-capacity cases.",
 		},
 		baseLabels,
 	)
 	spareCapacity = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: constants.WVASpareCapacity,
-			Help: "Per-variant spare capacity (0.0-1.0) from saturation analysis",
+			Help: "Per-variant spare KV-cache capacity (0.0-1.0) from saturation analysis. V1 path: threshold-relative spare (kvCacheThreshold - avg KV usage). V2 path: 1.0 - utilization.",
 		},
 		baseLabels,
 	)
 	requiredCapacity = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: constants.WVARequiredCapacity,
-			Help: "Model-level required capacity; >0 indicates scale-up needed. Use the analyzer_version label to distinguish units (V1: binary 0/1, V2: continuous token demand).",
+			Help: fmt.Sprintf("Model-level required capacity; >0 indicates scale-up needed. Use the %q label to interpret the value: %q → 0/1 scale-up signal (V1), %q → token demand (V2).", constants.LabelUnit, constants.UnitBinary, constants.UnitContinuous),
 		},
 		requiredCapacityLabels,
 	)
 	kvCacheTokensUsed = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: constants.WVAKvCacheTokensUsed,
-			Help: "Total KV cache tokens currently in use across all replicas of a variant",
+			Help: "Total KV cache tokens currently in use across all replicas of a variant (sum of vLLM TokensInUse).",
 		},
 		modelLabels,
 	)
-	kvCacheTokensTotal = prometheus.NewGaugeVec(
+	kvCacheTokensCapacity = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: constants.WVAKvCacheTokensTotal,
-			Help: "Total KV cache token capacity across all replicas of a variant",
+			Name: constants.WVAKvCacheTokensCapacity,
+			Help: "Total KV cache token capacity across all replicas of a variant (sum of vLLM TotalKvCapacityTokens).",
 		},
 		modelLabels,
 	)
@@ -151,8 +151,8 @@ func InitMetrics(registry prometheus.Registerer) error {
 	if err := registry.Register(kvCacheTokensUsed); err != nil {
 		return fmt.Errorf("failed to register kvCacheTokensUsed metric: %w", err)
 	}
-	if err := registry.Register(kvCacheTokensTotal); err != nil {
-		return fmt.Errorf("failed to register kvCacheTokensTotal metric: %w", err)
+	if err := registry.Register(kvCacheTokensCapacity); err != nil {
+		return fmt.Errorf("failed to register kvCacheTokensCapacity metric: %w", err)
 	}
 
 	return nil
@@ -230,16 +230,16 @@ func (m *MetricsEmitter) EmitReplicaMetrics(ctx context.Context, va *llmdOptv1al
 }
 
 // EmitSaturationMetrics emits saturation analysis and KV cache capacity metrics.
-// analyzerVersion ("v1" or "v2") is used as a label on wva_required_capacity to
-// disambiguate the units of the required value (V1: binary, V2: continuous tokens).
+// requiredCapacityUnit ("binary" or "continuous") is used as the "unit" label on
+// wva_required_capacity to describe how the value should be interpreted.
 func (m *MetricsEmitter) EmitSaturationMetrics(
 	ctx context.Context,
-	variantName, namespace, acceleratorType, analyzerVersion string,
+	variantName, namespace, acceleratorType, requiredCapacityUnit string,
 	utilization, spare, required float64,
-	kvTokensUsed, kvTokensTotal int64,
+	kvTokensUsed, kvTokensCapacity int64,
 ) error {
 	if saturationUtilization == nil || spareCapacity == nil || requiredCapacity == nil ||
-		kvCacheTokensUsed == nil || kvCacheTokensTotal == nil {
+		kvCacheTokensUsed == nil || kvCacheTokensCapacity == nil {
 		return errors.New("saturation metrics not initialized")
 	}
 
@@ -253,9 +253,9 @@ func (m *MetricsEmitter) EmitSaturationMetrics(
 		constants.LabelNamespace:   namespace,
 	}
 	requiredLabels := prometheus.Labels{
-		constants.LabelVariantName:     variantName,
-		constants.LabelNamespace:       namespace,
-		constants.LabelAnalyzerVersion: analyzerVersion,
+		constants.LabelVariantName: variantName,
+		constants.LabelNamespace:   namespace,
+		constants.LabelUnit:        requiredCapacityUnit,
 	}
 
 	if controllerInstance != "" {
@@ -268,7 +268,7 @@ func (m *MetricsEmitter) EmitSaturationMetrics(
 	spareCapacity.With(accelLabels).Set(spare)
 	requiredCapacity.With(requiredLabels).Set(required)
 	kvCacheTokensUsed.With(modelLabels).Set(float64(kvTokensUsed))
-	kvCacheTokensTotal.With(modelLabels).Set(float64(kvTokensTotal))
+	kvCacheTokensCapacity.With(modelLabels).Set(float64(kvTokensCapacity))
 
 	return nil
 }
@@ -280,8 +280,42 @@ func (m *MetricsEmitter) EmitSaturationMetrics(
 // TODO: wire this from the controller's VariantAutoscaling delete handler / finalizer.
 // Until that wiring exists, deleted VAs leave their last-emitted metric values in the
 // registry indefinitely.
-func (m *MetricsEmitter) DeleteSaturationMetrics(variantName, namespace, acceleratorType, analyzerVersion string) {
-	if saturationUtilization == nil {
+// DeleteSaturationMetricsForVariant removes all saturation metric series for the
+// given (variant, namespace) regardless of accelerator type or unit label. Uses
+// Prometheus DeletePartialMatch so callers don't need to know every label value.
+//
+// Intended for two scenarios:
+//   - The optimization cycle produced no decision for this variant — existing
+//     series would otherwise persist with stale values until Prometheus' default
+//     5-minute staleness marker fires. Calling this ensures dashboards show a
+//     gap (honest "no fresh data") rather than stale values.
+//   - A VA is being removed (delete handler / finalizer) and the caller doesn't
+//     want to reconstruct the exact emitted label set.
+func (m *MetricsEmitter) DeleteSaturationMetricsForVariant(variantName, namespace string) {
+	if saturationUtilization == nil || spareCapacity == nil || requiredCapacity == nil ||
+		kvCacheTokensUsed == nil || kvCacheTokensCapacity == nil {
+		return
+	}
+	match := prometheus.Labels{
+		constants.LabelVariantName: variantName,
+		constants.LabelNamespace:   namespace,
+	}
+	if controllerInstance != "" {
+		match[constants.LabelControllerInstance] = controllerInstance
+	}
+	saturationUtilization.DeletePartialMatch(match)
+	spareCapacity.DeletePartialMatch(match)
+	requiredCapacity.DeletePartialMatch(match)
+	kvCacheTokensUsed.DeletePartialMatch(match)
+	kvCacheTokensCapacity.DeletePartialMatch(match)
+}
+
+func (m *MetricsEmitter) DeleteSaturationMetrics(variantName, namespace, acceleratorType, requiredCapacityUnit string) {
+	// No-op if any metric hasn't been initialized. InitMetrics registers all
+	// five together, so either all are non-nil or all are nil — checking every
+	// var defends against a future code path that partially initializes them.
+	if saturationUtilization == nil || spareCapacity == nil || requiredCapacity == nil ||
+		kvCacheTokensUsed == nil || kvCacheTokensCapacity == nil {
 		return
 	}
 	accelLabels := prometheus.Labels{
@@ -294,9 +328,9 @@ func (m *MetricsEmitter) DeleteSaturationMetrics(variantName, namespace, acceler
 		constants.LabelNamespace:   namespace,
 	}
 	requiredLabels := prometheus.Labels{
-		constants.LabelVariantName:     variantName,
-		constants.LabelNamespace:       namespace,
-		constants.LabelAnalyzerVersion: analyzerVersion,
+		constants.LabelVariantName: variantName,
+		constants.LabelNamespace:   namespace,
+		constants.LabelUnit:        requiredCapacityUnit,
 	}
 	if controllerInstance != "" {
 		accelLabels[constants.LabelControllerInstance] = controllerInstance
@@ -307,5 +341,5 @@ func (m *MetricsEmitter) DeleteSaturationMetrics(variantName, namespace, acceler
 	spareCapacity.Delete(accelLabels)
 	requiredCapacity.Delete(requiredLabels)
 	kvCacheTokensUsed.Delete(modelLabels)
-	kvCacheTokensTotal.Delete(modelLabels)
+	kvCacheTokensCapacity.Delete(modelLabels)
 }