Improve deployment experience and features (#76)

* Improve deployment experience and features

Two features are:
 1. Store terraform state in GCS buckets
 2. Support multiple deployments for different environments

Storing state in GCS buckets provides a canonical location for the
current deployment state, which better supports multiple maintainers.
Previously, the state was stored on the local developer machine. This
made life tedious and error prone for multiple maintainers, or even a
single maintainer with multiple machines.

Different environments allows us to avoid testing in prod, and to stage
releases properly. ci and prod are created for real world usage, and the
dev environment is an environment in which we can test out more risky
changes that won't be exposed to the outside world.

Note that these changes require terragrunt in addition to terraform.
This is a small open source tool that provides support for both of these
features. It breaks the chicken-and-egg loop of creating GCS buckets
using terraform, but needing a GCS bucket in which to store the state.
It also easily supports different live environments invoking the same
module with input parameters, allowing for clear code reuse while having
separate files for each environment. This is the least risky approach I
can see to supporting updates to one environment without modifying the
others.

* Link to install

.gitignore

@@ -1,2 +1,4 @@
 /deployment/.terraform
 /deployment/*.tfstate.backup
+/deployment/**/.terragrunt-cache
+/deployment/**/.terraform.lock.hcl

deployment/README.md

@@ -1,21 +1,24 @@
 # Distributor GCP Deployment
 
-This terraform script configures a GCP project with a running distributor.
+This directory contains configuration-as-code to deploy the distributor to GCP.
+ - `modules`: a terraform module to deploy the distributor
+ - `live`: terragrunt configurations for deploying to different environments
+
+Deploying this requires installation of:
+ - `terraform`
+ - [`terragrunt`](https://terragrunt.gruntwork.io/docs/getting-started/install/)
+
+## Deploying
 
 First authenticate via `gcloud` as a principle with sufficient ACLs for
 the project:
 ```bash
 gcloud auth application-default login
 ```
 
-Terraforming the project can be done with:
-```bash
-# Check that the project and region are right and edit if not!
-cat terraform.tfvars
-
-terraform init
-terraform apply
-```
+Terraforming the project can be done by:
+ 1. `cd` to the relevant `live` directory for the environment to change
+ 2. Run `terragrunt apply`
 
 This should bring up the DB and Distributor running in Cloud Run.
 
@@ -28,13 +31,15 @@ $DISTRIBUTOR_URI/distributor/v0/logs
 ```
 
 Note that you'll need to substitute the URI for the new instance that is
-created on apply. This is output from `terraform apply` under the name
+created on apply. This is output from `terragrunt apply` under the name
 `distributor_uri` but you can find this in the Cloud Run area of GCP if
 you lose it down the back of the sofa.
 
-- [ ] TODO(mhutchinson): Set up public URL mapping
-- [ ] TODO(mhutchinson): make the state be stored in a GCS bucket instead of
-      output into a local file.
+## Terraform State
+
+The state of the environments are stored in GCS buckets, which means that
+multiple collaborators can work together on the project without passing around
+terraform state files or risking stomping on each other's changes.
 
 ## Connecting to the DB
 

deployment/live/ci/terragrunt.hcl

@@ -0,0 +1,20 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_path_to_repo_root()}/deployment/modules/distributor"
+}
+
+locals {
+  common_vars = read_terragrunt_config(find_in_parent_folders())
+}
+
+inputs = merge(
+  local.common_vars.locals,
+  {
+    docker_tag = "latest"
+    env        = "ci"
+  }
+)
+

deployment/live/dev/terragrunt.hcl

@@ -0,0 +1,20 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_path_to_repo_root()}/deployment/modules/distributor"
+}
+
+locals {
+  common_vars = read_terragrunt_config(find_in_parent_folders())
+}
+
+inputs = merge(
+  local.common_vars.locals,
+  {
+    docker_tag = "latest"
+    env        = "dev"
+  }
+)
+

deployment/live/prod/terragrunt.hcl

@@ -0,0 +1,20 @@
+include {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "${get_path_to_repo_root()}/deployment/modules/distributor"
+}
+
+locals {
+  common_vars = read_terragrunt_config(find_in_parent_folders())
+}
+
+inputs = merge(
+  local.common_vars.locals,
+  {
+    docker_tag = "latest" # TODO(mhutchinson): change this to tagged releases
+    env        = "prod"
+  }
+)
+

deployment/live/terragrunt.hcl

@@ -0,0 +1,20 @@
+locals {
+  project_id  = "checkpoint-distributor"
+  region      = "us-central1"
+  env         = path_relative_to_include()
+}
+
+remote_state {
+  backend = "gcs"
+
+  config = {
+    project  = local.project_id
+    location = local.region
+    bucket   = "${local.project_id}-${local.env}-terraform-state"
+    prefix   = "${path_relative_to_include()}/terraform.tfstate"
+
+    gcs_bucket_labels = {
+      name  = "terraform_state_storage"
+    }
+  }
+}

deployment/main.tf → deployment/modules/distributor/main.tf

@@ -23,6 +23,11 @@ data "google_project" "project" {
   project_id = var.project_id
 }
 
+# This will be configured by terragrunt when deploying
+terraform {
+  backend "gcs" {}
+}
+
 # Enable Secret Manager API
 resource "google_project_service" "secretmanager_api" {
   service            = "secretmanager.googleapis.com"
@@ -51,7 +56,7 @@ resource "random_password" "db_user_pwd" {
   special = false
 }
 resource "google_secret_manager_secret" "dbpass" {
-  secret_id = "dbpasssecret"
+  secret_id = "dbpasssecret_${var.env}"
   replication {
     auto {}
   }
@@ -78,7 +83,7 @@ locals {
     "You cannot reuse an instance name for up to a week after you have deleted an instance."
     See https://cloud.google.com/sql/docs/mysql/delete-instance for details.
   */
-  network_name = "${var.network_name}-safer-${random_id.suffix.hex}"
+  network_name = "${var.network_name}-safer-${var.env}-${random_id.suffix.hex}"
 }
 
 ###
@@ -108,7 +113,7 @@ locals {
 
 module "safer-mysql-db" {
   source               = "GoogleCloudPlatform/sql-db/google//modules/safer_mysql"
-  name                 = "distributor-mysql-instance-1"
+  name                 = "distributor-mysql-${var.env}-instance-1"
   random_instance_name = true
   project_id           = var.project_id
 
@@ -140,13 +145,13 @@ module "safer-mysql-db" {
 ### Set up Cloud Run service
 ###
 resource "google_cloud_run_v2_service" "default" {
-  name         = "distributor-service"
-  location     = "us-central1"
-  launch_stage = "BETA"
+  name         = "distributor-service-${var.env}"
+  location     = var.region
+  launch_stage = "GA"
 
   template {
     containers {
-      image = "gcr.io/trillian-opensource-ci/distributor:latest" # Image to deploy
+      image = "gcr.io/trillian-opensource-ci/distributor:${var.docker_tag}" # Image to deploy
       args = [
         "--logtostderr",
         "--v=1",

deployment/variables.tf → deployment/modules/distributor/variables.tf

@@ -16,10 +16,22 @@
 
 variable "project_id" {
   description = "The project ID to host the cluster in"
+  type        = string
 }
 
 variable "region" {
   description = "The region to host the cluster in"
+  type        = string
+}
+
+variable "env" {
+  description = "Unique identifier for the env, e.g. ci or prod"
+  type        = string
+}
+
+variable "docker_tag" {
+  description = "The tag to use for docker images to be deployed, e.g. latest"
+  type        = string
 }
 
 variable "network_name" {