chore(halo/attest): add pending blocks metric (#3176)

Adds a `halo_attest_pending_blocks` metric that tracks the number of
consensus blocks between the first vote and reaching quorum. This will
alert if we loose xchain vote quorum. It will also allow tracking fast
vs slow validators (to a limited extent).

issue: none

halo/attest/keeper/keeper.go

@@ -313,6 +313,13 @@ func (k *Keeper) Approve(ctx context.Context, valset ValSet) error {
 			return errors.Wrap(err, "get att signatures")
 		}
 
+		{
+			// Calculate pending blocks; safe to ignore errors since metrics is non-critical.
+			current, _ := umath.ToUint64(sdk.UnwrapSDKContext(ctx).BlockHeight())
+			delta := umath.SubtractOrZero(current, att.GetCreatedHeight())
+			pendingBlocks.WithLabelValues(chainVerName).Set(float64(delta))
+		}
+
 		setMetrics := func(att *Attestation) {
 			approvedHeight.WithLabelValues(chainVerName).Set(float64(att.GetBlockHeight()))
 			approvedOffset.WithLabelValues(chainVerName).Set(float64(att.GetAttestOffset()))

halo/attest/keeper/metrics.go

@@ -85,6 +85,15 @@ var (
 		Name:      "votes_expected_total",
 		Help:      "Total number of expected votes for attestations per validator per chain version.",
 	}, []string{"validator", "chain_version"})
+
+	// pendingBlocks tracks the number of blocks between the first vote and quorum for the latest attestation
+	// Alert if growing (not reaching quorum).
+	pendingBlocks = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "halo",
+		Subsystem: "attest",
+		Name:      "pending_blocks",
+		Help:      "Number of blocks the latest attestation is/was pending per chain version",
+	}, []string{"chain_version"})
 )
 
 func latency(method string) func() {