feat: migrate from prometheus to victoriametrics

hosts/idols-aquamarine/prometheus/README.md → hosts/idols-aquamarine/victoriametrics/README.md

@@ -1,4 +1,4 @@
-# Prometheus & Alertmanager
+# Monitoring & Alerting
 
 ## Alert Rules
 

hosts/idols-aquamarine/victoriametrics/alertmanager.nix

@@ -0,0 +1,47 @@
+{config, ...}: {
+  services.prometheus.alertmanager = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9093;
+    webExternalUrl = "http://alertmanager.writefor.fun";
+    logLevel = "info";
+
+    environmentFile = config.age.secrets."alertmanager.env".path;
+    configuration = {
+      global = {
+        # The smarthost and SMTP sender used for mail notifications.
+        smtp_smarthost = "smtp.qq.com:465";
+        smtp_from = "$SMTP_SENDER_EMAIL";
+        smtp_auth_username = "$SMTP_AUTH_USERNAME";
+        smtp_auth_password = "$SMTP_AUTH_PASSWORD";
+        # smtp.qq.com:465 support SSL only, so we need to disable TLS here.
+        # https://service.mail.qq.com/detail/0/310
+        smtp_require_tls = false;
+      };
+      route = {
+        receiver = "default";
+        routes = [
+          {
+            group_by = ["host"];
+            group_wait = "5m";
+            group_interval = "5m";
+            repeat_interval = "4h";
+            receiver = "default";
+          }
+        ];
+      };
+      receivers = [
+        {
+          name = "default";
+          email_configs = [
+            {
+              to = "[email protected]";
+              # Whether to notify about resolved alerts.
+              send_resolved = true;
+            }
+          ];
+        }
+      ];
+    };
+  };
+}

hosts/idols-aquamarine/victoriametrics/default.nix

@@ -0,0 +1,4 @@
+{mylib, ...}: {
+  imports =
+    mylib.scanPaths ./.;
+}

hosts/idols-aquamarine/victoriametrics/values.nix

@@ -0,0 +1,132 @@
+{
+  lib,
+  myvars,
+  ...
+}: {
+  # Workaround for victoriametrics to store data in another place
+  # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
+  systemd.tmpfiles.rules = [
+    "D /data/apps/victoriametrics 0751 victoriametrics victoriametrics - -"
+    "L+ /var/lib/victoriametrics - - - - /data/apps/victoriametrics"
+  ];
+
+  # https://victoriametrics.io/docs/victoriametrics/latest/configuration/configuration/
+  services.victoriametrics = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    port = 9090;
+    retentionTime = "30d";
+
+    extraFlags = [
+      # Allowed percent of system memory VictoriaMetrics caches may occupy.
+      "-memory.allowedPercent 50"
+    ];
+    # Directory below /var/lib to store victoriametrics metrics data.
+    stateDir = "victoriametrics";
+
+    # specifies a set of targets and parameters describing how to scrape metrics from them.
+    # https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
+    scrapeConfigs =
+      [
+        # --- Homelab Applications --- #
+
+        {
+          job_name = "dnsmasq-exporter";
+          scrape_interval = "30s";
+          metrics_path = "/metrics";
+          static_configs = [
+            {
+              targets = ["${myvars.networking.hostsAddr.suzi.ipv4}:9153"];
+              labels.type = "app";
+              labels.app = "dnsmasq";
+              labels.host = "suzi";
+            }
+          ];
+        }
+
+        {
+          job_name = "v2ray-exporter";
+          scrape_interval = "30s";
+          metrics_path = "/metrics";
+          static_configs = [
+            {
+              targets = ["${myvars.networking.hostsAddr.aquamarine.ipv4}:9153"];
+              labels.type = "app";
+              labels.app = "v2ray";
+              labels.host = "aquamarine";
+            }
+          ];
+        }
+        {
+          job_name = "postgres-exporter";
+          scrape_interval = "30s";
+          metrics_path = "/metrics";
+          static_configs = [
+            {
+              targets = ["${myvars.networking.hostsAddr.aquamarine.ipv4}:9187"];
+              labels.type = "app";
+              labels.app = "postgresql";
+              labels.host = "aquamarine";
+            }
+          ];
+        }
+        {
+          job_name = "sftpgo-embedded-exporter";
+          scrape_interval = "30s";
+          metrics_path = "/metrics";
+          static_configs = [
+            {
+              targets = ["${myvars.networking.hostsAddr.aquamarine.ipv4}:10000"];
+              labels.type = "app";
+              labels.app = "sftpgo";
+              labels.host = "aquamarine";
+            }
+          ];
+        }
+      ]
+      # --- Hosts --- #
+      ++ (
+        lib.attrsets.foldlAttrs
+        (acc: hostname: addr:
+          acc
+          ++ [
+            {
+              job_name = "node-exporter-${hostname}";
+              scrape_interval = "30s";
+              metrics_path = "/metrics";
+              static_configs = [
+                {
+                  # All my NixOS hosts.
+                  targets = ["${addr.ipv4}:9100"];
+                  labels.type = "node";
+                  labels.host = hostname;
+                }
+              ];
+            }
+          ])
+        []
+        myvars.networking.hostsAddr
+      );
+  };
+
+  services.vmalert = {
+    enable = true;
+    settings = {
+      "datasource.url" = "http://localhost:9090";
+      "notifier.url" = "http://localhost:9093"; # alertmanager's api
+
+      # Whether to disable long-lived connections to the datasource.
+      "datasource.disableKeepAlive" = true;
+      # Whether to avoid stripping sensitive information such as auth headers or passwords
+      # from URLs in log messages or UI and exported metrics.
+      "datasource.showURL" = false;
+      rule = [
+        ./alert_rules/node-exporter.yml
+        ./alert_rules/kubestate-exporter.yml
+        ./alert_rules/etcd_embedded-exporter.yml
+        ./alert_rules/istio_embedded-exporter.yml
+        ./alert_rules/coredns_embedded-exporter.yml
+      ];
+    };
+  };
+}