Add new queue wait time metrics to dashboard

Author: ndr-ds

kubernetes/linera-validator/grafana-dashboards/linera/execution.json

@@ -1158,8 +1158,8 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.50, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))",
-          "legendFormat": "p50",
+          "expr": "histogram_quantile(0.50, sum(rate(linera_chain_worker_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le, queue_type))",
+          "legendFormat": "p50 {{queue_type}}",
           "range": true,
           "refId": "A"
         },
@@ -1169,8 +1169,8 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.90, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))",
-          "legendFormat": "p90",
+          "expr": "histogram_quantile(0.90, sum(rate(linera_chain_worker_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le, queue_type))",
+          "legendFormat": "p90 {{queue_type}}",
           "range": true,
           "refId": "B"
         },
@@ -1180,13 +1180,13 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))",
-          "legendFormat": "p99",
+          "expr": "histogram_quantile(0.99, sum(rate(linera_chain_worker_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le, queue_type))",
+          "legendFormat": "p99 {{queue_type}}",
           "range": true,
           "refId": "C"
         }
       ],
-      "title": "Chain Worker Request Queue Wait Time",
+      "title": "Chain Worker Queue Wait Time",
       "type": "timeseries"
     },
     {

linera-core/src/chain_worker/actor.rs

@@ -551,11 +551,11 @@ where
 
                     while count < cross_chain_batch_size {
                         match Pin::new(&mut cross_chain_updates).next().now_or_never() {
-                            Some(Some((req, _span, enqueued_at))) => {
+                            Some(Some((req, _span, _enqueued_at))) => {
                                 #[cfg(with_metrics)]
                                 metrics::CHAIN_WORKER_QUEUE_WAIT_TIME
                                     .with_label_values(&["cross_chain_updates"])
-                                    .observe(enqueued_at.elapsed().as_millis() as f64);
+                                    .observe(_enqueued_at.elapsed().as_millis() as f64);
                                 updates.entry(req.origin).or_default().extend(req.bundles);
                                 callbacks_by_origin
                                     .entry(req.origin)
@@ -613,11 +613,11 @@ where
 
                     while count < cross_chain_batch_size {
                         match Pin::new(&mut confirmations).next().now_or_never() {
-                            Some(Some((req, _span, enqueued_at))) => {
+                            Some(Some((req, _span, _enqueued_at))) => {
                                 #[cfg(with_metrics)]
                                 metrics::CHAIN_WORKER_QUEUE_WAIT_TIME
                                     .with_label_values(&["confirmations"])
-                                    .observe(enqueued_at.elapsed().as_millis() as f64);
+                                    .observe(_enqueued_at.elapsed().as_millis() as f64);
                                 confirmations_map
                                     .entry(req.recipient)
                                     .and_modify(|h| *h = (*h).max(req.latest_height))
@@ -646,15 +646,15 @@ where
                 }
                 RequestType::Regular => {
                     for _ in 0..regular_batch_size {
-                        let Some(Some((request, span, enqueued_at))) =
+                        let Some(Some((request, span, _enqueued_at))) =
                             Pin::new(&mut requests).next().now_or_never()
                         else {
                             break;
                         };
                         #[cfg(with_metrics)]
                         metrics::CHAIN_WORKER_QUEUE_WAIT_TIME
                             .with_label_values(&["regular"])
-                            .observe(enqueued_at.elapsed().as_millis() as f64);
+                            .observe(_enqueued_at.elapsed().as_millis() as f64);
                         Box::pin(worker.handle_request(request))
                             .instrument(span)
                             .await;