diff --git a/kubernetes/linera-validator/grafana-dashboards/linera/execution.json b/kubernetes/linera-validator/grafana-dashboards/linera/execution.json index ecab71c3b3c9..45fb86c09065 100644 --- a/kubernetes/linera-validator/grafana-dashboards/linera/execution.json +++ b/kubernetes/linera-validator/grafana-dashboards/linera/execution.json @@ -15,6 +15,7 @@ } ] }, + "description": "Execution metrics for Linera validators including block execution latency, operation processing, message handling, and WebAssembly performance statistics.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, @@ -23,14 +24,27 @@ "panels": [ { "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 11, + "id": 1, "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], "title": "Latencies", "type": "row" }, @@ -39,6 +53,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to execute a block, including all operations and state transitions", "fieldConfig": { "defaults": { "color": { @@ -82,15 +97,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -101,15 +159,17 @@ "id": 2, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -119,21 +179,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_block_execution_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "interval": "", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_block_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_block_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_block_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Block Execution Latency p99 ms (1m)", + "title": "Block Execution Latency", "type": "timeseries" }, { @@ -141,6 +216,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to execute a single operation within a block", "fieldConfig": { "defaults": { "color": { @@ -184,15 +260,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -200,18 +319,20 @@ "x": 12, "y": 1 }, - "id": 4, + "id": 3, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -222,13 +343,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_block_execution_latency_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_operation_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_operation_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_operation_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Block Execution Latency p50 ms (1m)", + "title": "Operation Execution Latency", "type": "timeseries" }, { @@ -236,6 +379,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to execute a cross-chain message", "fieldConfig": { "defaults": { "color": { @@ -279,15 +423,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -295,18 +482,20 @@ "x": 0, "y": 9 }, - "id": 21, + "id": 4, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -316,20 +505,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_operation_execution_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_message_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_message_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_message_execution_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Operation Execution Latency p99 ms (1m)", + "title": "Message Execution Latency", "type": "timeseries" }, { @@ -337,6 +542,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to decompress WebAssembly bytecode before execution", "fieldConfig": { "defaults": { "color": { @@ -380,15 +586,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -396,18 +645,20 @@ "x": 12, "y": 9 }, - "id": 22, + "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -417,20 +668,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.50, sum(rate(linera_operation_execution_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_bytecode_decompression_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_bytecode_decompression_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_bytecode_decompression_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Operation Execution Latency p50 ms (1m)", + "title": "Bytecode Decompression Latency", "type": "timeseries" }, { @@ -438,6 +705,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to compute cryptographic hash of chain state", "fieldConfig": { "defaults": { "color": { @@ -481,15 +749,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -497,18 +808,20 @@ "x": 0, "y": 17 }, - "id": 23, + "id": 6, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -518,20 +831,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_message_execution_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_state_hash_computation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_state_hash_computation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_state_hash_computation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Message Execution Latency p99 ms (1m)", + "title": "State Hash Computation Latency", "type": "timeseries" }, { @@ -539,6 +868,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time messages spend waiting in the cross-chain message queue", "fieldConfig": { "defaults": { "color": { @@ -582,15 +912,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -598,18 +971,20 @@ "x": 12, "y": 17 }, - "id": 24, + "id": 7, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -619,20 +994,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.50, sum(rate(linera_message_execution_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_cross_chain_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_cross_chain_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_cross_chain_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Message Execution Latency p50 ms (1m)", + "title": "Cross Chain Queue Wait Time", "type": "timeseries" }, { @@ -640,6 +1031,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time requests spend waiting in the chain worker processing queue", "fieldConfig": { "defaults": { "color": { @@ -683,15 +1075,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -699,18 +1134,20 @@ "x": 0, "y": 25 }, - "id": 18, + "id": 8, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -721,13 +1158,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_bytecode_decompression_latency_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_chain_worker_request_queue_wait_time_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Bytecode decompression latency p99 (1m)", + "title": "Chain Worker Request Queue Wait Time", "type": "timeseries" }, { @@ -735,6 +1194,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to create and process network initialization actions", "fieldConfig": { "defaults": { "color": { @@ -778,35 +1238,80 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + }, + "unit": "ms" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.6.0-83314", "targets": [ @@ -816,13 +1321,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_bytecode_decompression_latency_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_create_network_actions_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_create_network_actions_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_create_network_actions_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Bytecode decompression latency p50 (1m)", + "title": "Create Network Actions Latency", "type": "timeseries" }, { @@ -830,6 +1357,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Ratio of cache hits to total cache accesses for value cache", "fieldConfig": { "defaults": { "color": { @@ -879,7 +1407,8 @@ "value": 80 } ] - } + }, + "unit": "percentunit" }, "overrides": [] }, @@ -889,7 +1418,7 @@ "x": 0, "y": 33 }, - "id": 34, + "id": 56, "options": { "legend": { "calcs": [], @@ -898,12 +1427,10 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -911,20 +1438,48 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_state_hash_computation_latency_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "sum by (validator) (rate(linera_value_cache_hit{validator=~\"$validator\"}[1m]))\n/\n(\n sum by (validator) (rate(linera_value_cache_hit{validator=~\"$validator\"}[1m]))\n+ (\n sum by (validator) (rate(linera_value_cache_miss{validator=~\"$validator\"}[1m]))\n or on (validator) vector(0)\n )\n)", + "instant": false, + "legendFormat": "{{validator}} - Cache Hit Ratio", "range": true, "refId": "A" } ], - "title": "State hash computation latency p99 (1m)", + "title": "Value Cache Hit Ratio", "type": "timeseries" }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 10, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], + "title": "Rates", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which new chains are being opened in the network", "fieldConfig": { "defaults": { "color": { @@ -968,23 +1523,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 42 }, - "id": 35, + "id": 11, "options": { "legend": { "calcs": [], @@ -993,9 +1545,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1006,33 +1557,32 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_state_hash_computation_latency_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "sum(rate(linera_open_chain_count{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(linera_open_chain_count{validator=~\"$validator\"}[1m]))", + "legendFormat": "Total", + "range": true, + "refId": "B" } ], - "title": "State hash computation latency p50 (1m)", + "title": "Chains Opened per Second (1m avg)", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 12, - "panels": [], - "title": "Rates", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which blocks are being added to chains", "fieldConfig": { "defaults": { "color": { @@ -1076,23 +1626,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 42 }, - "id": 56, + "id": 12, "options": { "legend": { "calcs": [], @@ -1101,40 +1648,36 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_open_chain_count[1m])) by (pod)", - "legendFormat": "__auto", - "queryType": "range", + "expr": "sum(rate(linera_num_blocks{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, { "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" + "type": "prometheus", + "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_open_chain_count[1m]))", - "hide": false, + "expr": "sum(rate(linera_num_blocks{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", - "queryType": "range", "range": true, "refId": "B" } ], - "title": "Chains opened per second (averaged for the past minute)", + "title": "Blocks Added to Chains per Second (1m avg)", "type": "timeseries" }, { @@ -1142,6 +1685,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which blocks are being executed by validators", "fieldConfig": { "defaults": { "color": { @@ -1185,23 +1729,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 42 + "x": 0, + "y": 50 }, - "id": 57, + "id": 13, "options": { "legend": { "calcs": [], @@ -1210,9 +1751,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1222,34 +1762,25 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(linera_num_blocks[1m])) by (pod)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "expr": "sum(rate(linera_num_blocks_executed{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(linera_num_blocks[1m]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "expr": "sum(rate(linera_num_blocks_executed{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" } ], - "title": "Blocks added to chains per second (averaged for the past minute)", + "title": "Blocks Executed per Second (1m avg)", "type": "timeseries" }, { @@ -1257,6 +1788,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of transaction processing across the network", "fieldConfig": { "defaults": { "color": { @@ -1300,23 +1832,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 50 }, - "id": 7, + "id": 14, "options": { "legend": { "calcs": [], @@ -1325,9 +1854,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1338,8 +1866,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_num_blocks_executed[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_transaction_count{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1349,14 +1877,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_num_blocks_executed[1m]))", - "hide": false, + "expr": "sum(rate(linera_transaction_count{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Blocks executed per second (averaged for the past minute)", + "title": "Transactions per Second (1m avg)", "type": "timeseries" }, { @@ -1364,6 +1891,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of individual operations being executed", "fieldConfig": { "defaults": { "color": { @@ -1407,23 +1935,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 58 }, - "id": 59, + "id": 15, "options": { "legend": { "calcs": [], @@ -1432,9 +1957,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1445,8 +1969,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_transaction_count[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_operation_count{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1456,14 +1980,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_transaction_count[1m]))", - "hide": false, + "expr": "sum(rate(linera_operation_count{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Transactions per second (averaged for the past minute)", + "title": "Operations per Second (1m avg)", "type": "timeseries" }, { @@ -1471,6 +1994,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of incoming message bundles from other chains", "fieldConfig": { "defaults": { "color": { @@ -1514,23 +2038,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 58 }, - "id": 60, + "id": 16, "options": { "legend": { "calcs": [], @@ -1539,9 +2060,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1552,8 +2072,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_operation_count[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_incoming_bundle_count{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1563,14 +2083,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_operation_count[1m]))", - "hide": false, + "expr": "sum(rate(linera_incoming_bundle_count{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Operations per second (averaged for the past minute)", + "title": "Incoming Bundles per Second (1m avg)", "type": "timeseries" }, { @@ -1578,6 +2097,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which the cross-chain message channel reaches capacity", "fieldConfig": { "defaults": { "color": { @@ -1621,23 +2141,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 58 + "x": 0, + "y": 66 }, - "id": 61, + "id": 18, "options": { "legend": { "calcs": [], @@ -1646,9 +2163,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1659,8 +2175,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_incoming_bundle_count[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_cross_chain_message_channel_full{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1670,14 +2186,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_incoming_bundle_count[1m]))", - "hide": false, + "expr": "sum(rate(linera_cross_chain_message_channel_full{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Incoming bundles per second (averaged for the past minute)", + "title": "Cross Chain Message Channel Full per Second (1m avg)", "type": "timeseries" }, { @@ -1685,6 +2200,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which the cross-chain message channel reaches capacity", "fieldConfig": { "defaults": { "color": { @@ -1728,23 +2244,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 66 }, - "id": 55, + "id": 59, "options": { "legend": { "calcs": [], @@ -1753,9 +2266,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1765,34 +2277,25 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(linera_cross_chain_message_tasks) by (pod)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "expr": "sum(rate(linera_notifications_skipped_receiver_lag{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(linera_cross_chain_message_tasks)", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "expr": "sum(rate(linera_notifications_skipped_receiver_lag{validator=~\"$validator\"}[1m]))", "legendFormat": "Total", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" } ], - "title": "Concurrent cross chain message tasks", + "title": "Notifications Skipped - Receiver lag, per Second (1m avg)", "type": "timeseries" }, { @@ -1800,6 +2303,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which the cross-chain message channel reaches capacity", "fieldConfig": { "defaults": { "color": { @@ -1843,23 +2347,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 66 + "x": 0, + "y": 74 }, - "id": 62, + "id": 60, "options": { "legend": { "calcs": [], @@ -1868,9 +2369,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1880,18 +2380,25 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "rate(linera_notification_channel_full[1m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "expr": "sum(rate(linera_notifications_dropped_no_receiver{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(linera_notifications_dropped_no_receiver{validator=~\"$validator\"}[1m]))", + "legendFormat": "Total", + "range": true, + "refId": "B" } ], - "title": "Notification channel full per second (averaged for the past minute)", + "title": "Notifications dropped - No receiver, per Second (1m avg)", "type": "timeseries" }, { @@ -1899,6 +2406,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate at which each validator is signing certificates", "fieldConfig": { "defaults": { "color": { @@ -1942,23 +2450,20 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 74 }, - "id": 9, + "id": 19, "options": { "legend": { "calcs": [], @@ -1967,9 +2472,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1980,20 +2484,58 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_certificates_signed[1m])) by (validator_name)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_certificates_signed_count{validator=~\"$validator\"}[1m])) by (validator)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(linera_certificates_signed_count{validator=~\"$validator\"}[1m]))", + "legendFormat": "Total", + "range": true, + "refId": "B" } ], - "title": "Certificates signed per second per validator (averaged for the past minute)", + "title": "Certificates Signed per Second per Validator (1m avg)", "type": "timeseries" }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 21, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], + "title": "Other", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Number of inbox queues maintained by each chain", "fieldConfig": { "defaults": { "color": { @@ -2037,34 +2579,79 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 74 + "x": 0, + "y": 83 }, - "id": 54, + "id": 24, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2075,13 +2662,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(linera_cross_chain_message_channel_full[1m])", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_num_inboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_num_inboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_num_inboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Cross chain message channel full per second (averaged for the past minute)", + "title": "Number of Inboxes", "type": "timeseries" }, { @@ -2089,6 +2698,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of outbox queues maintained by each chain", "fieldConfig": { "defaults": { "color": { @@ -2132,36 +2742,53 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "short" }, "overrides": [ { - "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Cache hit pct %" - ], - "prefix": "All except:", - "readOnly": true + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" }, "properties": [ { - "id": "custom.hideFrom", + "id": "color", "value": { - "legend": false, - "tooltip": false, - "viz": true + "fixedColor": "red", + "mode": "fixed" } } ] @@ -2172,20 +2799,22 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 83 }, - "id": 14, + "id": 25, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2196,33 +2825,43 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "((\n sum(rate(linera_value_cache_hit[1m])) by (key_type, value_type) or vector(0)\n)\n/\n(\n sum(rate(linera_value_cache_hit[1m])) by (key_type, value_type) or vector(0)\n +\n sum(rate(linera_value_cache_miss[1m])) by (key_type, value_type) or vector(0)\n)) * 100", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_num_outboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_num_outboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_num_outboxes_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Value cache hit ratio", + "title": "Number of Outboxes", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 90 - }, - "id": 13, - "panels": [], - "title": "Other", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Number of messages pending in chain inbox queues", "fieldConfig": { "defaults": { "color": { @@ -2266,57 +2905,118 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 91 - }, - "id": 40, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + }, + "unit": "short" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0-83314", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_num_rounds_in_certificate_bucket[1m])) by (le, certificate_value, round_type))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{certificate_value}} - {{round_type}}", + "expr": "histogram_quantile(0.50, sum(rate(linera_inbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_inbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_inbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Rounds in certificate p99 ms (1m)", + "title": "Inbox Size", "type": "timeseries" }, { @@ -2324,6 +3024,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of messages pending in chain outbox queues", "fieldConfig": { "defaults": { "color": { @@ -2367,15 +3068,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -2383,18 +3127,20 @@ "x": 12, "y": 91 }, - "id": 41, + "id": 27, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2405,13 +3151,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_num_rounds_in_certificate_bucket[1m])) by (le, certificate_value, round_type))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_outbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_outbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_outbox_size_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Rounds in certificate p50 ms (1m)", + "title": "Outbox Size", "type": "timeseries" }, { @@ -2419,6 +3187,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of message bundles removed during processing", "fieldConfig": { "defaults": { "color": { @@ -2462,15 +3231,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -2478,18 +3290,20 @@ "x": 0, "y": 99 }, - "id": 44, + "id": 28, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2499,20 +3313,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_num_inboxes_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_removed_bundles_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_removed_bundles_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_removed_bundles_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Number of inboxes p99 ms (1m)", + "title": "Removed Bundles", "type": "timeseries" }, { @@ -2520,6 +3350,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of consensus rounds in block proposal phase", "fieldConfig": { "defaults": { "color": { @@ -2563,15 +3394,58 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -2579,18 +3453,20 @@ "x": 12, "y": 99 }, - "id": 45, + "id": 29, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2601,1846 +3477,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_num_inboxes_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_num_rounds_in_block_proposal_bucket{validator=~\"$validator\"}[1m])) by (le, round_type))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Number of inboxes p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 107 - }, - "id": 46, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_num_outboxes_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Number of outboxes p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 107 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_num_outboxes_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Number of outboxes p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 115 - }, - "id": 48, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_inbox_size_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Inbox size p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 115 - }, - "id": 49, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_inbox_size_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Inbox size p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 123 - }, - "id": 52, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_outbox_size_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Outbox size p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 123 - }, - "id": 53, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_outbox_size_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Outbox size p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 131 - }, - "id": 50, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_removed_bundles_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Removed bundles p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 131 - }, - "id": 51, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_removed_bundles_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Removed bundles p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 139 - }, - "id": 42, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_num_rounds_in_block_proposal_bucket[1m])) by (le, round_type))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Rounds in block proposal p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 139 - }, - "id": 43, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_num_rounds_in_block_proposal_bucket[1m])) by (le, round_type))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Rounds in block proposal p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 147 - }, - "id": 10, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Value" - } - ] - }, - "pluginVersion": "10.1.1", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "sum(linera_certificates_signed) by (validator_name)", - "format": "table", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Certificates signed per validator", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 155 - }, - "id": 25, - "panels": [], - "title": "Wasm", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 156 - }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_contract_instantiation_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Wasm Contract Instantiation Latency p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 156 - }, - "id": 37, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_contract_instantiation_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Wasm Contract Instantiation Latency p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 164 - }, - "id": 38, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_service_instantiation_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Wasm Service Instantiation Latency p99 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 164 - }, - "id": 39, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_service_instantiation_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{pod}} - {{le}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Wasm Service Instantiation Latency p50 ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 172 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_fuel_used_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Wasm fuel used per block p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 172 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_fuel_used_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Wasm fuel used per block p50 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 180 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_num_reads_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.90, sum(rate(linera_num_rounds_in_block_proposal_bucket{validator=~\"$validator\"}[1m])) by (le, round_type))", + "legendFormat": "p90", "range": true, - "refId": "A" - } - ], - "title": "Wasm num reads per block p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 180 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_num_reads_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum(rate(linera_num_rounds_in_block_proposal_bucket{validator=~\"$validator\"}[1m])) by (le, round_type))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Wasm num reads per block p50 (1m)", + "title": "Rounds in Block Proposal", "type": "timeseries" }, { @@ -4448,6 +3513,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of consensus rounds required to create a certificate", "fieldConfig": { "defaults": { "color": { @@ -4489,35 +3555,81 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 188 + "y": 107 }, - "id": 30, + "id": 23, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -4528,107 +3640,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_bytes_read_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_num_rounds_in_certificate_bucket{validator=~\"$validator\"}[1m])) by (le, certificate_value, round_type))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Wasm bytes read per block p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 188 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_num_rounds_in_certificate_bucket{validator=~\"$validator\"}[1m])) by (le, certificate_value, round_type))", + "legendFormat": "p90", + "range": true, + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_bytes_read_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum(rate(linera_num_rounds_in_certificate_bucket{validator=~\"$validator\"}[1m])) by (le, certificate_value, round_type))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Wasm bytes read per block p50 (1m)", + "title": "Rounds in Certificate", "type": "timeseries" }, { @@ -4677,7 +3717,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4691,10 +3732,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 196 + "x": 12, + "y": 107 }, - "id": 32, + "id": 55, "options": { "legend": { "calcs": [], @@ -4715,14 +3756,34 @@ "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_bytes_written_per_block_bucket[1m])) by (le))", + "expr": "sum(linera_cross_chain_message_tasks{validator=~\"$validator\"}) by (pod)", + "fullMetaSearch": false, + "includeNullMetadata": true, "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(linera_cross_chain_message_tasks{validator=~\"$validator\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Total", + "range": true, + "refId": "B", + "useBackend": false } ], - "title": "Wasm bytes written per block p99 (1m)", + "title": "Concurrent cross chain message tasks", "type": "timeseries" }, { @@ -4730,6 +3791,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total number of certificates signed by each validator", "fieldConfig": { "defaults": { "color": { @@ -4771,24 +3833,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 196 + "x": 0, + "y": 115 }, - "id": 33, + "id": 30, "options": { "legend": { "calcs": [], @@ -4797,9 +3857,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -4810,20 +3869,47 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_bytes_written_per_block_bucket[1m])) by (le))", - "legendFormat": "__auto", + "expr": "linera_certificates_signed_count{validator=~\"$validator\"}", + "legendFormat": "{{validator}}", "range": true, "refId": "A" } ], - "title": "Wasm bytes written per block p50 (1m)", + "title": "Certificates Signed per Validator", "type": "timeseries" }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 123 + }, + "id": 31, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], + "title": "Wasm", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to instantiate a WebAssembly contract module", "fieldConfig": { "defaults": { "color": { @@ -4867,35 +3953,79 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 196 + "y": 124 }, - "id": 62, + "id": 32, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -4905,18 +4035,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_cross_chain_queue_wait_time_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_contract_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_contract_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_contract_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Cross chain queue wait time p99 (1m)", + "title": "Wasm Contract Instantiation Latency", "type": "timeseries" }, { @@ -4924,6 +4072,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time taken to instantiate a WebAssembly service module", "fieldConfig": { "defaults": { "color": { @@ -4967,35 +4116,79 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 196 + "y": 124 }, - "id": 63, + "id": 33, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -5005,18 +4198,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(linera_cross_chain_queue_wait_time_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_service_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_service_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_service_instantiation_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Cross chain queue wait time p50 (1m)", + "title": "Wasm Service Instantiation Latency", "type": "timeseries" }, { @@ -5024,6 +4235,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Amount of WebAssembly execution fuel consumed per block", "fieldConfig": { "defaults": { "color": { @@ -5067,35 +4279,79 @@ { "color": "green", "value": null - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] - }, - "unit": "ms" - }, - "overrides": [] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 132 }, - "id": 64, + "id": 34, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -5105,18 +4361,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_chain_worker_request_queue_wait_time_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_fuel_used_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_fuel_used_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_fuel_used_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Chain worker request queue wait time p99 (1m)", + "title": "Wasm Fuel Used per Block", "type": "timeseries" }, { @@ -5124,6 +4398,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Number of storage read operations during WebAssembly execution per block", "fieldConfig": { "defaults": { "color": { @@ -5167,35 +4442,79 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 132 }, - "id": 65, + "id": 35, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -5205,18 +4524,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(linera_chain_worker_request_queue_wait_time_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_num_reads_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_num_reads_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_num_reads_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Chain worker request queue wait time p50 (1m)", + "title": "Wasm Num Reads per Block", "type": "timeseries" }, { @@ -5224,6 +4561,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total bytes read from storage during WebAssembly execution per block", "fieldConfig": { "defaults": { "color": { @@ -5267,35 +4605,79 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 140 }, - "id": 66, + "id": 36, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -5305,18 +4687,36 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_create_network_actions_latency_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_bytes_read_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_bytes_read_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_bytes_read_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Create network actions latency p99 (1m)", + "title": "Wasm Bytes Read per Block", "type": "timeseries" }, { @@ -5324,6 +4724,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total bytes written to storage during WebAssembly execution per block", "fieldConfig": { "defaults": { "color": { @@ -5367,35 +4768,79 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 212 + "y": 140 }, - "id": 67, + "id": 37, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -5405,36 +4850,86 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_create_network_actions_latency_bucket[1m])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(linera_wasm_bytes_written_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p50", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_wasm_bytes_written_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(linera_wasm_bytes_written_per_block_bucket{validator=~\"$validator\"}[1m])) by (le))", + "legendFormat": "p99", + "range": true, + "refId": "C" } ], - "title": "Create network actions latency p50 (1m)", + "title": "Wasm Bytes Written per Block", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": [ + "linera" + ], "templating": { - "list": [] + "list": [ + { + "allFormat": "glob", + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(linera_block_execution_latency_bucket, validator)", + "hide": 0, + "includeAll": true, + "label": "Validator", + "multi": false, + "name": "validator", + "options": [], + "query": { + "query": "label_values(linera_block_execution_latency_bucket, validator)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, - "timezone": "browser", + "timezone": "", "title": "Execution", "uid": "aeeg4vnzeku80d", - "version": 2, + "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/kubernetes/linera-validator/grafana-dashboards/linera/general.json b/kubernetes/linera-validator/grafana-dashboards/linera/general.json index bdbf41e4c5a2..67044665f156 100644 --- a/kubernetes/linera-validator/grafana-dashboards/linera/general.json +++ b/kubernetes/linera-validator/grafana-dashboards/linera/general.json @@ -15,6 +15,7 @@ } ] }, + "description": "Comprehensive monitoring dashboard for Linera validator infrastructure including proxy metrics, server performance, and cluster resource utilization", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, @@ -39,6 +40,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of incoming requests to the proxy layer, aggregated per pod", "fieldConfig": { "defaults": { "color": { @@ -89,7 +91,7 @@ } ] }, - "unit": "none" + "unit": "reqps" }, "overrides": [] }, @@ -109,8 +111,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -121,7 +123,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_proxy_request_count[1m])) by (pod)", + "expr": "sum(rate(linera_proxy_request_count{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "__auto", "range": true, "refId": "A" @@ -135,6 +137,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Percentage of failed proxy requests over 1 minute intervals", "fieldConfig": { "defaults": { "color": { @@ -205,8 +208,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -219,7 +222,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "(\n (\n rate(linera_proxy_request_error[1m])\n ) / (\n (\n (\n rate(linera_proxy_request_success[1m])\n ) + (\n rate(linera_proxy_request_error[1m])\n )\n )\n )\n) * 100", + "expr": "(\n (\n rate(linera_proxy_request_error{validator=~\"$validator\"}[1m])\n ) / (\n (\n (\n rate(linera_proxy_request_success{validator=~\"$validator\"}[1m])\n ) + (\n rate(linera_proxy_request_error{validator=~\"$validator\"}[1m])\n )\n )\n )\n) * 100", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -238,6 +241,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Request latency percentiles for the proxy layer over 1 minute intervals", "fieldConfig": { "defaults": { "color": { @@ -290,7 +294,53 @@ }, "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, @@ -307,9 +357,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -321,212 +370,42 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_proxy_request_latency_bucket[1m])))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_proxy_request_latency_bucket{validator=~\"$validator\"}[1m])))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "p99", "range": true, "refId": "A", "useBackend": false - } - ], - "title": "Proxy p99 latency ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(linera_proxy_request_latency_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(linera_proxy_request_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", "instant": false, - "legendFormat": "__auto", + "legendFormat": "p90", "range": true, - "refId": "A" - } - ], - "title": "Proxy p90 latency ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_proxy_request_latency_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(linera_proxy_request_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", "instant": false, - "legendFormat": "__auto", + "legendFormat": "p50", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Proxy p50 latency ms (1m)", + "title": "Proxy Latency (ms)", "type": "timeseries" }, { @@ -535,7 +414,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 17 }, "id": 15, "panels": [], @@ -547,6 +426,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of incoming requests to the validator server, aggregated per pod", "fieldConfig": { "defaults": { "color": { @@ -596,411 +476,8 @@ "value": 80 } ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(rate(linera_server_request_count[1m])) by (pod)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(rate(linera_server_request_count[1m]))", - "hide": false, - "legendFormat": "Total", - "range": true, - "refId": "B" - } - ], - "title": "Server requests per second (averaged for the past minute)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "(\n (\n sum(rate(linera_server_request_error[1m])) by (method_name, pod)\n ) / (\n (\n (\n sum(rate(linera_server_request_success[1m])) by (method_name, pod)\n ) + (\n sum(rate(linera_server_request_error[1m])) by (method_name, pod)\n )\n )\n )\n) * 100", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{pod}} - {{method_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Server Error rate (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_server_request_latency_bucket[1m])) by (le))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Server p99 latency ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(linera_server_request_latency_bucket[1m])) by (le))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Server p90 latency ms (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] }, - "unit": "ms" + "unit": "reqps" }, "overrides": [] }, @@ -1008,9 +485,9 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 18 }, - "id": 17, + "id": 22, "options": { "legend": { "calcs": [], @@ -1020,8 +497,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1032,14 +509,25 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_server_request_latency_bucket[1m])) by (le))", - "instant": false, + "expr": "sum(rate(linera_server_request_count{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "__auto", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(linera_server_request_count{validator=~\"$validator\"}[1m]))", + "hide": false, + "legendFormat": "Total", + "range": true, + "refId": "B" } ], - "title": "Server p50 latency ms (1m)", + "title": "Server requests per second (averaged for the past minute)", "type": "timeseries" }, { @@ -1047,6 +535,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Percentage of failed server requests over 1 minute intervals", "fieldConfig": { "defaults": { "color": { @@ -1088,7 +577,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1096,7 +586,7 @@ } ] }, - "unit": "ms" + "unit": "percent" }, "overrides": [] }, @@ -1104,9 +594,9 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 18 }, - "id": 23, + "id": 12, "options": { "legend": { "calcs": [], @@ -1116,8 +606,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1129,17 +619,17 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(linera_server_request_latency_per_request_type_bucket[1m])) by (le, method_name))", + "expr": "(\n (\n sum(rate(linera_server_request_error{validator=~\"$validator\"}[1m])) by (method_name, pod)\n ) / (\n (\n (\n sum(rate(linera_server_request_success{validator=~\"$validator\"}[1m])) by (method_name, pod)\n ) + (\n sum(rate(linera_server_request_error{validator=~\"$validator\"}[1m])) by (method_name, pod)\n )\n )\n )\n) * 100", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{pod}} - {{method_name}}", "range": true, "refId": "A", "useBackend": false } ], - "title": "Server p99 latency per request type ms (1m)", + "title": "Server Error rate (1m)", "type": "timeseries" }, { @@ -1147,6 +637,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Request latency percentiles for the server over 1 minute intervals", "fieldConfig": { "defaults": { "color": { @@ -1188,7 +679,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1198,15 +690,61 @@ }, "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 26 }, - "id": 24, + "id": 13, "options": { "legend": { "calcs": [], @@ -1215,9 +753,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1229,17 +766,41 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(linera_server_request_latency_per_request_type_bucket[1m])) by (le, method_name))", + "expr": "histogram_quantile(0.99, sum(rate(linera_server_request_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "p99", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_server_request_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_server_request_latency_bucket{validator=~\"$validator\"}[1m])) by (le))", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C" } ], - "title": "Server p90 latency per request type ms (1m)", + "title": "Server Latency (ms)", "type": "timeseries" }, { @@ -1247,6 +808,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Request latency percentiles broken down by request type over 1 minute intervals", "fieldConfig": { "defaults": { "color": { @@ -1288,7 +850,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1298,15 +861,61 @@ }, "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 26 }, - "id": 25, + "id": 23, "options": { "legend": { "calcs": [], @@ -1315,9 +924,8 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1329,17 +937,49 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(linera_server_request_latency_per_request_type_bucket[1m])) by (le, method_name))", + "expr": "histogram_quantile(0.99, sum(rate(linera_server_request_latency_per_request_type_bucket{validator=~\"$validator\"}[1m])) by (le, method_name))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "p99", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(linera_server_request_latency_per_request_type_bucket{validator=~\"$validator\"}[1m])) by (le, method_name))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(linera_server_request_latency_per_request_type_bucket{validator=~\"$validator\"}[1m])) by (le, method_name))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "C", + "useBackend": false } ], - "title": "Server p50 latency per request type ms (1m)", + "title": "Server Latency by Request Type (ms)", "type": "timeseries" }, { @@ -1348,7 +988,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 34 }, "id": 8, "panels": [], @@ -1360,6 +1000,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total transmitted network bytes per validator pod", "fieldConfig": { "defaults": { "color": { @@ -1401,7 +1042,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1417,7 +1059,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 35 }, "id": 9, "options": { @@ -1429,8 +1071,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1441,7 +1083,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", + "expr": "sum(rate(container_network_transmit_bytes_total{validator=~\"$validator\",pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", "hide": false, "instant": false, "legendFormat": "tx - {{pod}}", @@ -1454,7 +1096,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{pod=~\"shards-.*\"}[60s]))", + "expr": "sum(rate(container_network_transmit_bytes_total{validator=~\"$validator\",pod=~\"shards-.*\"}[60s]))", "hide": false, "instant": false, "legendFormat": "tx - Total shards", @@ -1470,6 +1112,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total received network bytes per validator pod", "fieldConfig": { "defaults": { "color": { @@ -1511,7 +1154,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1527,7 +1171,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 35 }, "id": 47, "options": { @@ -1539,8 +1183,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1551,11 +1195,12 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", + "expr": "sum(rate(container_network_receive_bytes_total{validator=~\"$validator\",pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", + "hide": false, "instant": false, - "legendFormat": "rx - {{pod}}", + "legendFormat": "tx - {{pod}}", "range": true, - "refId": "A" + "refId": "C" }, { "datasource": { @@ -1563,12 +1208,12 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{pod=~\"shards-.*\"}[60s]))", + "expr": "sum(rate(container_network_receive_bytes_total{validator=~\"$validator\",pod=~\"shards-.*\"}[60s]))", "hide": false, "instant": false, - "legendFormat": "rx - Total shards", + "legendFormat": "tx - Total shards", "range": true, - "refId": "B" + "refId": "D" } ], "title": "Network bytes RX total per pod", @@ -1579,6 +1224,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Termination events per validator container", "fieldConfig": { "defaults": { "color": { @@ -1599,7 +1245,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "smooth", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1620,7 +1266,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1628,7 +1275,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -1636,9 +1283,9 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 43 }, - "id": 36, + "id": 39, "options": { "legend": { "calcs": [], @@ -1647,12 +1294,11 @@ "showLegend": true }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", + "pluginVersion": "10.1.1", "targets": [ { "datasource": { @@ -1660,40 +1306,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{container=~\"linera-.*\"}[60s])) by (container)", + "expr": "sum(rate(kube_pod_container_status_last_terminated_reason{validator=~\"$validator\"}[1m])) by (pod, reason)", "hide": false, "instant": false, - "legendFormat": "{{container}}", + "legendFormat": "{{pod}} - {{reason}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total[60s]))", - "hide": false, - "instant": false, - "legendFormat": "Total", - "range": true, - "refId": "C" } ], - "title": "Container CPU Usage", + "title": "Container termination", + "transformations": [], "type": "timeseries" }, { @@ -1701,6 +1323,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Exit codes from terminated validator containers", "fieldConfig": { "defaults": { "color": { @@ -1742,15 +1365,15 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, @@ -1758,25 +1381,23 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 43 }, - "id": 5, + "id": 46, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "none", - "sort": "asc" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", + "pluginVersion": "10.1.1", "targets": [ { "datasource": { @@ -1784,48 +1405,18 @@ "uid": "prometheus" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(container_memory_usage_bytes{container=~\"linera.*\"}) by (container)", + "editorMode": "builder", + "expr": "kube_pod_container_status_last_terminated_exitcode{validator=~\"$validator\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{container}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(container_memory_usage_bytes{pod=~\"(proxy|shards).*\"}) by (pod)", - "hide": false, - "instant": false, "legendFormat": "{{pod}}", "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(container_memory_usage_bytes{container=~\"linera.*\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Total", - "range": true, - "refId": "C", + "refId": "A", "useBackend": false } ], - "title": "Container Memory Usage Bytes", + "title": "Container termination exit code", "type": "timeseries" }, { @@ -1833,6 +1424,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "CPU utilization per validator container", "fieldConfig": { "defaults": { "color": { @@ -1874,7 +1466,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1882,7 +1475,7 @@ } ] }, - "unit": "none" + "unit": "percent" }, "overrides": [] }, @@ -1890,22 +1483,20 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 51 }, - "id": 29, + "id": 36, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "none", - "sort": "asc" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -1915,19 +1506,41 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(pod, failure_type) (rate(container_memory_failures_total{pod=~\"(proxy|shards)-.*\", failure_type!=\"pgfault\"}[60s]))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{validator=~\"$validator\",pod=~\"(proxy|shards)-.*\"}[60s])) by (pod)", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{validator=~\"$validator\",container=~\"linera-.*\"}[60s])) by (container)", + "hide": false, + "instant": false, + "legendFormat": "{{container}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{validator=~\"$validator\"}[60s]))", + "hide": false, "instant": false, - "legendFormat": "{{pod}} - {{failure_type}}", + "legendFormat": "Total", "range": true, - "refId": "A", - "useBackend": false + "refId": "C" } ], - "title": "Container Memory Failures", + "title": "Container CPU Usage", "type": "timeseries" }, { @@ -1935,7 +1548,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "", + "description": "Memory allocation failures per validator container", "fieldConfig": { "defaults": { "color": { @@ -1956,7 +1569,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1977,7 +1590,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1993,39 +1607,44 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 51 }, - "id": 39, + "id": 29, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "10.1.1", + "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(kube_pod_container_status_last_terminated_reason[1m])) by (pod, reason)", - "hide": false, + "expr": "sum by(pod, failure_type) (rate(container_memory_failures_total{validator=~\"$validator\",pod=~\"(proxy|shards)-.*\", failure_type!=\"pgfault\"}[60s]))", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "{{pod}} - {{reason}}", + "legendFormat": "{{pod}} - {{failure_type}}", "range": true, - "refId": "B" + "refId": "A", + "useBackend": false } ], - "title": "Container termination", - "transformations": [], + "title": "Container Memory Failures", "type": "timeseries" }, { @@ -2033,6 +1652,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem read operations per second for validator shards", "fieldConfig": { "defaults": { "color": { @@ -2074,7 +1694,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2082,7 +1703,7 @@ } ] }, - "unit": "Bps" + "unit": "iops" }, "overrides": [] }, @@ -2090,9 +1711,9 @@ "h": 8, "w": 12, "x": 0, - "y": 83 + "y": 59 }, - "id": 32, + "id": 41, "options": { "legend": { "calcs": [], @@ -2102,8 +1723,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2114,7 +1735,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_bytes_total{container=\"linera-server\"}[60s])) by (pod)", + "expr": "sum(rate(container_fs_reads_total{validator=~\"$validator\",container=\"linera-server\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -2126,7 +1747,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_bytes_total{container=\"linera-server\"}[60s]))", + "expr": "sum(rate(container_fs_reads_total{validator=~\"$validator\",container=\"linera-server\"}[60s]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -2134,7 +1755,7 @@ "refId": "B" } ], - "title": "Shards FS read bytes per second", + "title": "Shards FS reads per second", "type": "timeseries" }, { @@ -2142,7 +1763,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Some useful codes to remember:\n137 - SIGKILL - can happen on OOMs\n143 - SIGTERM", + "description": "Memory usage in bytes per validator container", "fieldConfig": { "defaults": { "color": { @@ -2184,14 +1805,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, @@ -2199,23 +1822,25 @@ "h": 8, "w": 12, "x": 12, - "y": 83 + "y": 59 }, - "id": 46, + "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "10.1.1", + "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -2223,18 +1848,48 @@ "uid": "prometheus" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "kube_pod_container_status_last_terminated_exitcode", + "editorMode": "code", + "expr": "sum(container_memory_usage_bytes{validator=~\"$validator\",container=~\"linera.*\"}) by (container)", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{container}}", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(container_memory_usage_bytes{validator=~\"$validator\",pod=~\"(proxy|shards).*\"}) by (pod)", + "hide": false, + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(container_memory_usage_bytes{validator=~\"$validator\",container=~\"linera.*\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "C", + "useBackend": false } ], - "title": "Container termination exit code", + "title": "Container Memory Usage Bytes", "type": "timeseries" }, { @@ -2242,6 +1897,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem read throughput in bytes per second for validator shards", "fieldConfig": { "defaults": { "color": { @@ -2283,7 +1939,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2299,9 +1956,9 @@ "h": 8, "w": 12, "x": 0, - "y": 91 + "y": 67 }, - "id": 44, + "id": 32, "options": { "legend": { "calcs": [], @@ -2311,8 +1968,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2323,7 +1980,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_bytes_total{container=\"linera-server\"}[60s])) by (pod)", + "expr": "sum(rate(container_fs_reads_bytes_total{validator=~\"$validator\",container=\"linera-server\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -2335,7 +1992,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_bytes_total{container=\"linera-server\"}[60s]))", + "expr": "sum(rate(container_fs_reads_bytes_total{validator=~\"$validator\",container=\"linera-server\"}[60s]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -2343,7 +2000,7 @@ "refId": "B" } ], - "title": "Shards FS written bytes per second", + "title": "Shards FS read bytes per second", "type": "timeseries" }, { @@ -2351,6 +2008,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem write operations per second for validator shards", "fieldConfig": { "defaults": { "color": { @@ -2392,7 +2050,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2400,7 +2059,7 @@ } ] }, - "unit": "none" + "unit": "iops" }, "overrides": [] }, @@ -2408,9 +2067,9 @@ "h": 8, "w": 12, "x": 12, - "y": 91 + "y": 67 }, - "id": 41, + "id": 31, "options": { "legend": { "calcs": [], @@ -2420,8 +2079,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2432,7 +2091,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_total{container=\"linera-server\"}[60s])) by (pod)", + "expr": "sum(rate(container_fs_writes_total{validator=~\"$validator\",container=\"linera-server\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -2444,7 +2103,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_total{container=\"linera-server\"}[60s]))", + "expr": "sum(rate(container_fs_writes_total{validator=~\"$validator\",container=\"linera-server\"}[60s]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -2452,7 +2111,7 @@ "refId": "B" } ], - "title": "Shards FS reads per second", + "title": "Shards FS writes per second", "type": "timeseries" }, { @@ -2460,6 +2119,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem write throughput in bytes per second for validator shards", "fieldConfig": { "defaults": { "color": { @@ -2501,7 +2161,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2509,17 +2170,17 @@ } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 99 + "x": 0, + "y": 75 }, - "id": 31, + "id": 44, "options": { "legend": { "calcs": [], @@ -2529,8 +2190,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2541,7 +2202,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_total{container=\"linera-server\"}[60s])) by (pod)", + "expr": "sum(rate(container_fs_writes_bytes_total{validator=~\"$validator\",container=\"linera-server\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -2553,7 +2214,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_total{container=\"linera-server\"}[60s]))", + "expr": "sum(rate(container_fs_writes_bytes_total{validator=~\"$validator\",container=\"linera-server\"}[60s]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -2561,7 +2222,7 @@ "refId": "B" } ], - "title": "Shards FS writes per second", + "title": "Shards FS written bytes per second", "type": "timeseries" }, { @@ -2570,7 +2231,7 @@ "h": 1, "w": 24, "x": 0, - "y": 107 + "y": 83 }, "id": 34, "panels": [], @@ -2582,6 +2243,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total transmitted network bytes for ScyllaDB pods", "fieldConfig": { "defaults": { "color": { @@ -2623,7 +2285,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2639,7 +2302,7 @@ "h": 8, "w": 12, "x": 0, - "y": 108 + "y": 84 }, "id": 30, "options": { @@ -2651,8 +2314,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2663,7 +2326,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{pod=~\"scylla-validator.*\"}[60s])) by (pod)", + "expr": "sum(rate(container_network_transmit_bytes_total{validator=~\"$validator\",pod=~\"scylla-validator.*\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{container}}", "range": true, @@ -2678,6 +2341,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Total received network bytes for ScyllaDB pods", "fieldConfig": { "defaults": { "color": { @@ -2719,7 +2383,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2735,7 +2400,7 @@ "h": 8, "w": 12, "x": 12, - "y": 108 + "y": 84 }, "id": 48, "options": { @@ -2747,8 +2412,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2759,7 +2424,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{pod=~\"scylla-validator.*\"}[60s])) by (pod)", + "expr": "sum(rate(container_network_receive_bytes_total{validator=~\"$validator\",pod=~\"scylla-validator.*\"}[60s])) by (pod)", "instant": false, "legendFormat": "{{container}}", "range": true, @@ -2774,6 +2439,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Termination events for ScyllaDB containers", "fieldConfig": { "defaults": { "color": { @@ -2815,7 +2481,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2823,7 +2490,7 @@ } ] }, - "unit": "percentunit" + "unit": "none" }, "overrides": [] }, @@ -2831,9 +2498,9 @@ "h": 8, "w": 12, "x": 0, - "y": 116 + "y": 92 }, - "id": 26, + "id": 45, "options": { "legend": { "calcs": [], @@ -2843,8 +2510,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -2854,15 +2521,19 @@ "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"scylla-validator-.*\"}[60s])) by (pod)", + "expr": "sum(rate(kube_pod_container_status_last_terminated_reason{validator=~\"$validator\",validator=~\"$validator\",pod=~\"(scylla)-.*\"}[1m])) by (pod, reason)", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "{{container}}", + "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "ScyllaDB Total CPU Usage", + "title": "ScyllaDB Container termination", "type": "timeseries" }, { @@ -2870,6 +2541,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Memory allocation failures for ScyllaDB containers", "fieldConfig": { "defaults": { "color": { @@ -2911,7 +2583,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2919,7 +2592,7 @@ } ] }, - "unit": "decbytes" + "unit": "none" }, "overrides": [] }, @@ -2927,9 +2600,9 @@ "h": 8, "w": 12, "x": 12, - "y": 116 + "y": 92 }, - "id": 10, + "id": 27, "options": { "legend": { "calcs": [ @@ -2941,11 +2614,11 @@ }, "tooltip": { "hideZeros": false, - "mode": "none", - "sort": "asc" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "10.1.1", + "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -2954,7 +2627,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum(container_memory_usage_bytes{pod=~\"scylla-validator.*\"}) by (pod)", + "expr": "sum by(failure_type) (rate(container_memory_failures_total{validator=~\"$validator\",container=\"scylla\", failure_type!=\"pgfault\"}[60s]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -2964,7 +2637,7 @@ "useBackend": false } ], - "title": "Scylla Memory Usage Bytes", + "title": "Scylla Memory Failures", "type": "timeseries" }, { @@ -2972,7 +2645,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "", + "description": "CPU utilization for ScyllaDB containers", "fieldConfig": { "defaults": { "color": { @@ -3014,7 +2687,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3022,7 +2696,7 @@ } ] }, - "unit": "none" + "unit": "percent" }, "overrides": [] }, @@ -3030,9 +2704,9 @@ "h": 8, "w": 12, "x": 0, - "y": 124 + "y": 100 }, - "id": 45, + "id": 26, "options": { "legend": { "calcs": [], @@ -3042,8 +2716,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3053,19 +2727,15 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(kube_pod_container_status_last_terminated_reason{pod=~\"(scylla)-.*\"}[1m])) by (pod, reason)", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{validator=~\"$validator\",pod=~\"scylla-validator-.*\"}[60s])) by (pod)", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{container}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "ScyllaDB Container termination", + "title": "ScyllaDB Total CPU Usage", "type": "timeseries" }, { @@ -3073,6 +2743,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Out-of-memory events per second for ScyllaDB containers", "fieldConfig": { "defaults": { "color": { @@ -3114,7 +2785,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3130,22 +2802,20 @@ "h": 8, "w": 12, "x": 12, - "y": 124 + "y": 100 }, - "id": 27, + "id": 38, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "none", - "sort": "asc" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3157,7 +2827,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(failure_type) (rate(container_memory_failures_total{container=\"scylla\", failure_type!=\"pgfault\"}[60s]))", + "expr": "sum(rate(container_oom_events_total{validator=~\"$validator\",pod=~\"scylla-validator-.*\"}[1m])) by (pod)", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -3167,7 +2837,7 @@ "useBackend": false } ], - "title": "Scylla Memory Failures", + "title": "ScyllaDB OOM events per second", "type": "timeseries" }, { @@ -3175,6 +2845,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem read operations per second for ScyllaDB", "fieldConfig": { "defaults": { "color": { @@ -3216,7 +2887,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3224,7 +2896,7 @@ } ] }, - "unit": "none" + "unit": "iops" }, "overrides": [] }, @@ -3232,7 +2904,7 @@ "h": 8, "w": 12, "x": 0, - "y": 132 + "y": 108 }, "id": 28, "options": { @@ -3244,8 +2916,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3256,7 +2928,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_total{container=\"scylla\"}[60s])) by (container)", + "expr": "sum(rate(container_fs_reads_total{validator=~\"$validator\",container=\"scylla\"}[60s])) by (container)", "instant": false, "legendFormat": "{{container}}", "range": true, @@ -3271,6 +2943,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Memory usage in bytes for ScyllaDB containers", "fieldConfig": { "defaults": { "color": { @@ -3312,7 +2985,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3320,7 +2994,7 @@ } ] }, - "unit": "Bps" + "unit": "decbytes" }, "overrides": [] }, @@ -3328,38 +3002,44 @@ "h": 8, "w": 12, "x": 12, - "y": 132 + "y": 108 }, - "id": 42, + "id": 10, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", + "pluginVersion": "10.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(container_fs_reads_bytes_total{container=\"scylla\"}[60s])) by (container)", + "expr": "sum(container_memory_usage_bytes{validator=~\"$validator\",pod=~\"scylla-validator.*\"}) by (pod)", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "{{container}}", + "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "ScyllaDB FS read bytes per second", + "title": "Scylla Memory Usage Bytes", "type": "timeseries" }, { @@ -3367,6 +3047,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem read throughput in bytes per second for ScyllaDB", "fieldConfig": { "defaults": { "color": { @@ -3408,7 +3089,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3416,7 +3098,7 @@ } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, @@ -3424,9 +3106,9 @@ "h": 8, "w": 12, "x": 0, - "y": 140 + "y": 116 }, - "id": 43, + "id": 42, "options": { "legend": { "calcs": [], @@ -3436,8 +3118,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3448,14 +3130,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_total{container=\"scylla\"}[60s])) by (container)", + "expr": "sum(rate(container_fs_reads_bytes_total{validator=~\"$validator\",container=\"scylla\"}[60s])) by (container)", "instant": false, "legendFormat": "{{container}}", "range": true, "refId": "A" } ], - "title": "ScyllaDB FS writes per second", + "title": "ScyllaDB FS read bytes per second", "type": "timeseries" }, { @@ -3463,6 +3145,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem write operations per second for ScyllaDB", "fieldConfig": { "defaults": { "color": { @@ -3504,7 +3187,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3512,7 +3196,7 @@ } ] }, - "unit": "Bps" + "unit": "iops" }, "overrides": [] }, @@ -3520,9 +3204,9 @@ "h": 8, "w": 12, "x": 12, - "y": 140 + "y": 116 }, - "id": 33, + "id": 43, "options": { "legend": { "calcs": [], @@ -3532,8 +3216,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3544,14 +3228,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(container_fs_writes_bytes_total{container=\"scylla\"}[60s])) by (container)", + "expr": "sum(rate(container_fs_writes_total{validator=~\"$validator\",container=\"scylla\"}[60s])) by (container)", "instant": false, "legendFormat": "{{container}}", "range": true, "refId": "A" } ], - "title": "ScyllaDB FS written bytes per second", + "title": "ScyllaDB FS writes per second", "type": "timeseries" }, { @@ -3559,6 +3243,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Filesystem write throughput in bytes per second for ScyllaDB", "fieldConfig": { "defaults": { "color": { @@ -3600,7 +3285,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3608,7 +3294,7 @@ } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, @@ -3616,9 +3302,9 @@ "h": 8, "w": 12, "x": 0, - "y": 148 + "y": 124 }, - "id": 38, + "id": 33, "options": { "legend": { "calcs": [], @@ -3628,8 +3314,8 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "11.6.0-83314", @@ -3639,19 +3325,15 @@ "type": "prometheus", "uid": "prometheus" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(rate(container_oom_events_total{pod=~\"scylla-validator-.*\"}[1m])) by (pod)", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "sum(rate(container_fs_writes_bytes_total{validator=~\"$validator\",container=\"scylla\"}[60s])) by (container)", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{container}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "ScyllaDB OOM events per second", + "title": "ScyllaDB FS written bytes per second", "type": "timeseries" } ], @@ -3662,7 +3344,33 @@ "linera" ], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Validator", + "multi": true, + "name": "validator", + "options": [], + "query": "label_values(linera_proxy_request_count, validator)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] }, "time": { "from": "now-1h", diff --git a/kubernetes/linera-validator/grafana-dashboards/linera/storage.json b/kubernetes/linera-validator/grafana-dashboards/linera/storage.json index 0aed7a8f212a..34c7fb440f9a 100644 --- a/kubernetes/linera-validator/grafana-dashboards/linera/storage.json +++ b/kubernetes/linera-validator/grafana-dashboards/linera/storage.json @@ -15,6 +15,7 @@ } ] }, + "description": "Storage layer metrics including latencies for loading views, chains, contracts, and services, plus I/O operation rates", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, @@ -39,6 +40,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time to load a view from storage. p50 is median, p90 is most users' experience, p99 captures tail latency.", "fieldConfig": { "defaults": { "color": { @@ -76,21 +78,55 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ { - "color": "green", - "value": null - }, + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, @@ -101,18 +137,17 @@ "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": ["mean", "max"], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -120,108 +155,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_view_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_view_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Load view latency p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(linera_load_view_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p90", + "range": true, + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_view_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_view_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Load view latency p50 (1m)", + "title": "Load View Latency", "type": "timeseries" }, { @@ -229,6 +191,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time to load chain state from storage. p50 is median, p90 is most users' experience, p99 captures tail latency.", "fieldConfig": { "defaults": { "color": { @@ -266,43 +229,76 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "green", - "value": null - }, + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 9 + "x": 12, + "y": 1 }, "id": 15, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": ["mean", "max"], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -310,108 +306,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_chain_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_chain_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Load chain latency p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(linera_load_chain_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p90", + "range": true, + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_chain_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_chain_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Load chain latency p50 (1m)", + "title": "Load Chain Latency", "type": "timeseries" }, { @@ -419,6 +342,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time to load a contract from storage. p50 is median, p90 is most users' experience, p99 captures tail latency.", "fieldConfig": { "defaults": { "color": { @@ -456,43 +380,76 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ { - "color": "green", - "value": null - }, + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 9 }, "id": 22, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": ["mean", "max"], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -500,108 +457,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_contract_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_contract_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Load contract latency p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(linera_load_contract_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p90", + "range": true, + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_contract_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_contract_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Load contract latency p50 (1m)", + "title": "Load Contract Latency", "type": "timeseries" }, { @@ -609,6 +493,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Time to load a service from storage. p50 is median, p90 is most users' experience, p99 captures tail latency.", "fieldConfig": { "defaults": { "color": { @@ -646,43 +531,76 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ { - "color": "green", - "value": null - }, + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p90" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 25 + "x": 12, + "y": 9 }, "id": 24, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": ["mean", "max"], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -690,108 +608,35 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_service_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_service_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p50", "range": true, "refId": "A" - } - ], - "title": "Load service latency p99 (1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by(le) (rate(linera_load_service_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p90", + "range": true, + "refId": "B" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0-83314", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(linera_load_service_latency_bucket[1m])))", - "legendFormat": "__auto", + "expr": "histogram_quantile(0.99, sum by(le) (rate(linera_load_service_latency_bucket{validator=~\"$validator\"}[1m])))", + "legendFormat": "p99", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Load service latency p50 (1m)", + "title": "Load Service Latency", "type": "timeseries" }, { @@ -800,7 +645,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 17 }, "id": 21, "panels": [], @@ -812,6 +657,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of contains_blobs operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -849,19 +695,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -869,7 +703,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 18 }, "id": 2, "options": { @@ -881,11 +715,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -893,13 +726,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_contains_blobs[1m])) by (pod)", + "expr": "sum(rate(linera_contains_blobs{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Contains blobs per second (averaged for the past minute)", + "title": "Contains Blobs Rate", "type": "timeseries" }, { @@ -907,6 +740,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of contains_blob_state operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -944,19 +778,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -964,7 +786,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 18 }, "id": 3, "options": { @@ -976,11 +798,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -988,13 +809,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_contains_blob_state[1m])) by (pod)", + "expr": "sum(rate(linera_contains_blob_state{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Contains blob state per second (averaged for the past minute)", + "title": "Contains Blob State Rate", "type": "timeseries" }, { @@ -1002,6 +823,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of event write operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1039,19 +861,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1059,7 +869,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 26 }, "id": 14, "options": { @@ -1071,11 +881,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1083,13 +892,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(linera_write_event[1m])", + "expr": "sum(rate(linera_write_event{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Write event per second (averaged for the past minute)", + "title": "Write Event Rate", "type": "timeseries" }, { @@ -1097,6 +906,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of event read operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1134,19 +944,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1154,7 +952,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 26 }, "id": 13, "options": { @@ -1166,11 +964,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1178,13 +975,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_event[1m])) by (pod)", + "expr": "sum(rate(linera_read_event{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read event per second (averaged for the past minute)", + "title": "Read Event Rate", "type": "timeseries" }, { @@ -1192,6 +989,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of certificate read operations (batch) per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1229,19 +1027,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1249,7 +1035,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 34 }, "id": 11, "options": { @@ -1261,11 +1047,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1273,8 +1058,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_certificates[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_read_certificates{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1284,14 +1069,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_certificates[1m]))", + "expr": "sum(rate(linera_read_certificates{validator=~\"$validator\"}[1m]))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Read certificates per second (averaged for the past minute)", + "title": "Read Certificates Rate (batch)", "type": "timeseries" }, { @@ -1299,6 +1084,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of single certificate read operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1336,19 +1122,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1356,7 +1130,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 34 }, "id": 10, "options": { @@ -1368,11 +1142,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1380,13 +1153,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_certificate[1m])) by (pod)", + "expr": "sum(rate(linera_read_certificate{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read certificate per second (averaged for the past minute)", + "title": "Read Certificate Rate (single)", "type": "timeseries" }, { @@ -1394,6 +1167,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of blob state read operations (batch) per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1431,19 +1205,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1451,7 +1213,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 42 }, "id": 8, "options": { @@ -1463,11 +1225,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1475,13 +1236,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_blob_states[1m])) by (pod)", + "expr": "sum(rate(linera_read_blob_states{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read blob states per second (averaged for the past minute)", + "title": "Read Blob States Rate (batch)", "type": "timeseries" }, { @@ -1489,6 +1250,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of single blob state read operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1526,19 +1288,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1546,7 +1296,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 42 }, "id": 7, "options": { @@ -1558,11 +1308,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1570,13 +1319,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_blob_state[1m])) by (pod)", + "expr": "sum(rate(linera_read_blob_state{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read blob state per second (averaged for the past minute)", + "title": "Read Blob State Rate (single)", "type": "timeseries" }, { @@ -1584,6 +1333,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of blob write operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1621,19 +1371,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1641,7 +1379,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 50 }, "id": 9, "options": { @@ -1653,11 +1391,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1665,13 +1402,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_write_blob[1m])) by (pod)", + "expr": "sum(rate(linera_write_blob{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Write blob per second (averaged for the last minute)", + "title": "Write Blob Rate", "type": "timeseries" }, { @@ -1679,6 +1416,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of blob read operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1716,19 +1454,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1736,7 +1462,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 50 }, "id": 6, "options": { @@ -1748,11 +1474,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1760,13 +1485,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_blob[1m])) by (pod)", + "expr": "sum(rate(linera_read_blob{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read blob per second (averaged for the past minute)", + "title": "Read Blob Rate", "type": "timeseries" }, { @@ -1774,6 +1499,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of certificate write operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1811,19 +1537,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1831,7 +1545,7 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 58 }, "id": 12, "options": { @@ -1843,11 +1557,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1855,8 +1568,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_write_certificate[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_write_certificate{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -1866,14 +1579,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_write_certificate[1m]))", + "expr": "sum(rate(linera_write_certificate{validator=~\"$validator\"}[1m]))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Write certificate per second (averaged for the past minute)", + "title": "Write Certificate Rate", "type": "timeseries" }, { @@ -1881,6 +1594,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of confirmed block read operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -1918,19 +1632,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -1938,7 +1640,7 @@ "h": 8, "w": 12, "x": 12, - "y": 74 + "y": 58 }, "id": 5, "options": { @@ -1950,11 +1652,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -1962,13 +1663,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_read_confirmed_block[1m])) by (pod)", + "expr": "sum(rate(linera_read_confirmed_block{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Read hashed confirmed block per second (averaged for the past minute)", + "title": "Read Confirmed Block Rate", "type": "timeseries" }, { @@ -1976,6 +1677,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of contains_blob operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -2013,19 +1715,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -2033,7 +1723,7 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 66 }, "id": 1, "options": { @@ -2045,11 +1735,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -2057,13 +1746,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_contains_blob[1m])) by (pod)", - "legendFormat": "{{[pod}}", + "expr": "sum(rate(linera_contains_blob{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Contains Blob per second (averaged for the past minute)", + "title": "Contains Blob Rate", "type": "timeseries" }, { @@ -2071,6 +1760,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of contains_certificate operations per second by pod.", "fieldConfig": { "defaults": { "color": { @@ -2108,19 +1798,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -2128,7 +1806,7 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 66 }, "id": 4, "options": { @@ -2140,11 +1818,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -2152,22 +1829,51 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_contains_certificate[1m])) by (pod)", + "expr": "sum(rate(linera_contains_certificate{validator=~\"$validator\"}[1m])) by (pod)", "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Contains certificate per second (averaged for the past minute)", + "title": "Contains Certificate Rate", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": ["linera"], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(linera_load_view_latency_bucket, validator)", + "hide": 0, + "includeAll": true, + "label": "Validator", + "multi": false, + "name": "validator", + "options": [], + "query": { + "query": "label_values(linera_load_view_latency_bucket, validator)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-1h", diff --git a/kubernetes/linera-validator/grafana-dashboards/linera/views.json b/kubernetes/linera-validator/grafana-dashboards/linera/views.json index 307a27b637c3..9c5d2c820c50 100644 --- a/kubernetes/linera-validator/grafana-dashboards/linera/views.json +++ b/kubernetes/linera-validator/grafana-dashboards/linera/views.json @@ -15,6 +15,7 @@ } ] }, + "description": "Metrics related to view loading, saving, and caching operations", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, @@ -26,6 +27,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of view save operations per second, broken down by pod. Higher values indicate more state changes being persisted.", "fieldConfig": { "defaults": { "color": { @@ -63,19 +65,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -95,11 +85,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -107,8 +96,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_save_view[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_save_view{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -118,14 +107,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_save_view[1m]))", + "expr": "sum(rate(linera_save_view{validator=~\"$validator\"}[1m]))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Save view per second (averaged for the past minute)", + "title": "Save View Rate", "type": "timeseries" }, { @@ -133,6 +122,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate of view load operations per second, broken down by pod. Higher values indicate more state being read from storage.", "fieldConfig": { "defaults": { "color": { @@ -170,19 +160,7 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "unit": "ops" }, "overrides": [] }, @@ -202,11 +180,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -214,8 +191,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_load_view[1m])) by (pod)", - "legendFormat": "__auto", + "expr": "sum(rate(linera_load_view{validator=~\"$validator\"}[1m])) by (pod)", + "legendFormat": "{{pod}}", "range": true, "refId": "A" }, @@ -225,14 +202,14 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(linera_load_view[1m])) ", + "expr": "sum(rate(linera_load_view{validator=~\"$validator\"}[1m]))", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], - "title": "Load view per second (averaged for the past minute)", + "title": "Load View Rate", "type": "timeseries" }, { @@ -240,6 +217,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Percentage of cache hits vs total cache accesses. Higher values indicate better cache efficiency and less storage I/O.", "fieldConfig": { "defaults": { "color": { @@ -277,19 +255,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percent" }, "overrides": [] @@ -310,11 +275,10 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.6.0-83314", "targets": [ { "datasource": { @@ -322,22 +286,51 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "(\n sum(rate(linera_num_cache_success[1m])) by (pod)\n /\n (\n sum(rate(linera_num_cache_success[1m])) by (pod)\n + sum(rate(linera_num_cache_fault[1m])) by (pod)\n )\n) * 100", - "legendFormat": "__auto", + "expr": "(\n sum(rate(linera_num_cache_success{validator=~\"$validator\"}[1m])) by (pod)\n /\n (\n sum(rate(linera_num_cache_success{validator=~\"$validator\"}[1m])) by (pod)\n + sum(rate(linera_num_cache_fault{validator=~\"$validator\"}[1m])) by (pod)\n )\n) * 100", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Cache hit ratio (1m)", + "title": "Cache Hit Ratio", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": ["linera"], "templating": { - "list": [] + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(linera_load_view, validator)", + "hide": 0, + "includeAll": true, + "label": "Validator", + "multi": false, + "name": "validator", + "options": [], + "query": { + "query": "label_values(linera_load_view, validator)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-1h", @@ -349,4 +342,4 @@ "uid": "aeeg8d3516zuof", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/kubernetes/linera-validator/grafana-dashboards/profiling/cpu.json b/kubernetes/linera-validator/grafana-dashboards/profiling/cpu.json index 018ccd0442c2..ac4d73345805 100644 --- a/kubernetes/linera-validator/grafana-dashboards/profiling/cpu.json +++ b/kubernetes/linera-validator/grafana-dashboards/profiling/cpu.json @@ -17,7 +17,7 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 2, "links": [], "liveNow": false, "panels": [ diff --git a/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json b/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json index c67f579228d0..351d008e1d52 100644 --- a/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json +++ b/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json @@ -17,7 +17,7 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 2, "links": [], "liveNow": false, "panels": [ diff --git a/linera-core/src/chain_worker/state.rs b/linera-core/src/chain_worker/state.rs index 8a83017af851..e14e8abea4ca 100644 --- a/linera-core/src/chain_worker/state.rs +++ b/linera-core/src/chain_worker/state.rs @@ -538,26 +538,50 @@ where })?; hashes.push(hash); } - let certificates = self.storage.read_certificates(hashes.clone()).await?; - let certificates = match ResultReadCertificates::new(certificates, hashes) { - ResultReadCertificates::Certificates(certificates) => certificates, - ResultReadCertificates::InvalidHashes(hashes) => { - return Err(WorkerError::ReadCertificatesError(hashes)) + + let mut uncached_hashes = Vec::new(); + let mut height_to_blocks: HashMap> = HashMap::new(); + + for hash in hashes { + if let Some(hashed_block) = self.block_values.get(&hash) { + height_to_blocks.insert(hashed_block.inner().header.height, hashed_block); + } else { + uncached_hashes.push(hash); } - }; - let height_to_certificates = heights - .into_iter() - .zip(certificates) - .collect::>(); - // For each medium, select the relevant messages. + } + + if !uncached_hashes.is_empty() { + let certificates = self + .storage + .read_certificates(uncached_hashes.clone()) + .await?; + let certificates = match ResultReadCertificates::new(certificates, uncached_hashes) { + ResultReadCertificates::Certificates(certificates) => certificates, + ResultReadCertificates::InvalidHashes(hashes) => { + return Err(WorkerError::ReadCertificatesError(hashes)) + } + }; + + for cert in certificates { + let hashed_block = cert.into_value().into_inner(); + let height = hashed_block.inner().header.height; + self.block_values.insert(Cow::Owned(hashed_block.clone())); + height_to_blocks.insert(height, hashed_block); + } + } + let mut cross_chain_requests = Vec::new(); for (recipient, heights) in heights_by_recipient { let mut bundles = Vec::new(); for height in heights { - let cert = height_to_certificates + let hashed_block = height_to_blocks .get(&height) - .ok_or_else(|| ChainError::InternalError("missing certificates".to_string()))?; - bundles.extend(cert.message_bundles_for(recipient)); + .ok_or_else(|| ChainError::InternalError("missing block".to_string()))?; + bundles.extend( + hashed_block + .inner() + .message_bundles_for(recipient, hashed_block.hash()), + ); } let request = CrossChainRequest::UpdateRecipient { sender: self.chain.chain_id(),