Define liveness alert policy and add its graph to the dashboard.

deployment/live/monitoring/prod/terragrunt.hcl

@@ -7,6 +7,7 @@ inputs = merge(
   include.root.locals,
   {
     alert_lt_num_witness_threshold = 10
+    alert_enable_num_witness = false
   }
 )
 

deployment/live/monitoring/terragrunt.hcl

@@ -3,9 +3,10 @@ terraform {
 }
 
 locals {
-  project_id  = "checkpoint-distributor"
-  region      = "us-central1"
-  env         = path_relative_to_include()
+  project_id               = "checkpoint-distributor"
+  region                   = "us-central1"
+  env                      = path_relative_to_include()
+  alert_enable_num_witness = true
 }
 
 remote_state {

deployment/modules/monitoring/main.tf

@@ -97,6 +97,7 @@ resource "google_monitoring_dashboard" "witness_dashboard" {
 }
 
 resource "google_monitoring_alert_policy" "witness_liveness" {
+  enabled = var.alert_enable_num_witness
   display_name = "Number of live witnesses (${var.env})"
   combiner     = "OR"
   conditions {

deployment/modules/monitoring/variables.tf

@@ -33,3 +33,8 @@ variable "alert_lt_num_witness_threshold" {
   description = "The lower bound alert threshold for the number of live witnesses, as measured by the distributor_update_checkpoint_success Prometheus metric."
   type        = number
 }
+
+variable "alert_enable_num_witness" {
+  description = "Whether to enable alert_lt_num_witness_threshold."
+  type        = bool
+}