Optimize tests (#46)

* Optimize TA tests * Update grafana dashboard config
kysre · Feb 13, 2024 · f4b015e · f4b015e
1 parent 479b125
commit f4b015e
Show file tree

Hide file tree

Showing 2 changed files with 132 additions and 29 deletions.
diff --git a/grafana/dashboard.json b/grafana/dashboard.json
@@ -863,22 +863,55 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
                 "value": 80
               }
             ]
           },
-          "unit": "decbytes",
+          "unit": "mps",
           "unitScale": true
         },
         "overrides": []
@@ -889,21 +922,20 @@
         "x": 0,
         "y": 32
       },
-      "id": 9,
+      "id": 13,
       "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
-        "orientation": "auto",
-        "reduceOptions": {
+        "legend": {
           "calcs": [
             "lastNotNull"
           ],
-          "fields": "",
-          "values": false
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
         },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "sizing": "auto"
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "10.3.1",
       "targets": [
@@ -913,15 +945,15 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(disk_used_size{}[1m])) by (provider)",
+          "expr": "sum(rate(message_count{}[1m])) by (provider) ",
           "instant": false,
           "legendFormat": "{{provider}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Datanode Disk Usage",
-      "type": "gauge"
+      "title": "Message Rate",
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -970,7 +1002,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1012,7 +1045,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(message_count{}[1m])) by (provider)",
+          "expr": "message_count{}",
           "instant": false,
           "legendFormat": "{{provider}}",
           "range": true,
@@ -1022,6 +1055,75 @@
       "title": "Message Per Datanode",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decbytes",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 9,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(disk_used_size{}[1m])) by (provider)",
+          "instant": false,
+          "legendFormat": "{{provider}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Datanode Disk Usage",
+      "type": "gauge"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -1069,7 +1171,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green"
+                "color": "green",
+                "value": null
               },
               {
                 "color": "red",
@@ -1085,7 +1188,7 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
+        "x": 12,
         "y": 40
       },
       "id": 12,
@@ -1130,8 +1233,8 @@
       {
         "current": {
           "selected": false,
-          "text": "0.99",
-          "value": "0.99"
+          "text": "0.9",
+          "value": "0.9"
         },
         "hide": 0,
         "includeAll": false,
@@ -1140,7 +1243,7 @@
         "name": "percentile",
         "options": [
           {
-            "selected": false,
+            "selected": true,
             "text": "0.9",
             "value": "0.9"
           },
@@ -1150,7 +1253,7 @@
             "value": "0.95"
           },
           {
-            "selected": true,
+            "selected": false,
             "text": "0.99",
             "value": "0.99"
           }
@@ -1163,13 +1266,13 @@
     ]
   },
   "time": {
-    "from": "now-15m",
+    "from": "now-5m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "TurtleMQ",
   "uid": "ce242e9f-8458-4ae5-b053-e9c81ef0de2f",
-  "version": 21,
+  "version": 23,
   "weekStart": ""
 }
diff --git a/test_fault_tolerance.py b/test_fault_tolerance.py
@@ -41,12 +41,12 @@ def store(_: str, val: bytes):
 print("press enter when cluster is healthy")
 input()
 
-for _ in range(SUBSCRIBER_COUNT):
-    subscribe(store)
-
 for i in range(TEST_SIZE // 2, TEST_SIZE):
     push(f"{key_seq[i]}", f"{i}".encode(encoding="utf-8"))
 
+for _ in range(SUBSCRIBER_COUNT):
+    subscribe(store)
+
 # Wait to consume all messages
 print("Wait for all messages to be consumed and press enter")
 input()