Merge pull request #1935 from cmu-delphi/release/indicators_v0.3.51_utils_v0.3.22

Release covidcast-indicators 0.3.51

Author: melange396

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.50
+current_version = 0.3.51
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.gitignore

@@ -129,6 +129,8 @@ venv.bak/
 .retry
 .indicators-ansible-vault-pass
 indicators-ansible-vault-pass
+vault_plain*
+vault_backup*
 
 # testing_utils
 testing_utils/cache

ansible/README.md

@@ -0,0 +1,43 @@
+# Ansible
+
+Ansible is used in the indicators deployment process.
+
+It aids in:
+
+- Setting up staging and production systems to run the indicators.
+- Placing templated indicators params files onto staging and production systems.
+- Managing secrets.
+
+## Managing secrets with ansible-vault
+
+The deployment process uses [`ansible-vault`](https://docs.ansible.com/ansible/latest/vault_guide/index.html) and a corresponding file of `vault.yaml` to write secrets into template files that are placed onto staging and production systems. `vault.yaml` should always be encrypted.
+
+To work with secrets in this repo you should follow one of these processes:
+
+1. Work with systems administrators to add secrets.
+
+OR
+
+2. Obtain the vault decryption password and use the helper scripts.
+
+- Make sure you are in the repo's `ansible` directory.
+
+  ```shell
+  cd $(git rev-parse --show-toplevel)/ansible
+  ```
+
+- Use the helper scripts to:
+
+  - Decrypt to `vault_plain.yaml` - Creates a .gitgnored "plain" file for editing. Also a backup directory and backup file if possible.
+
+    ```shell
+    bash vault-decrypt.sh
+    ```
+
+  - Make your changes in `vault_plain.yaml`
+
+  - Encrypt to a new `vault.yaml` - Creates a new encrypted vault file suitable for committing. Also creates a backup directory and backup file if possible.
+
+    ```shell
+    bash vault-encrypt.sh
+    ```

ansible/templates/changehc-params-prod.json.j2

@@ -62,10 +62,10 @@
   },
   "archive": {
     "aws_credentials": {
-      "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
-      "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+      "aws_access_key_id": "{{ archive_differ_bucket_user_access_key_id }}",
+      "aws_secret_access_key": "{{ archive_differ_bucket_user_secret_access_key }}"
     },
-    "bucket_name": "delphi-covidcast-indicator-output",
+    "bucket_name": "{{ archive_differ_bucket_name }}",
     "cache_dir": "./cache",
     "indicator_prefix": "delphi_changehc"
   }

ansible/templates/hhs_hosp-params-prod.json.j2

@@ -22,10 +22,10 @@
   },
   "archive": {
     "aws_credentials": {
-      "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
-      "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+      "aws_access_key_id": "{{ archive_differ_bucket_user_access_key_id }}",
+      "aws_secret_access_key": "{{ archive_differ_bucket_user_secret_access_key }}"
     },
-    "bucket_name": "delphi-covidcast-indicator-output",
+    "bucket_name": "{{ archive_differ_bucket_name }}",
     "cache_dir": "./cache",
     "indicator_prefix": "delphi_hhs_hosp"
   }

ansible/templates/nchs_mortality-params-prod.json.j2

@@ -11,10 +11,10 @@
   },
   "archive": {
     "aws_credentials": {
-      "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
-      "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+      "aws_access_key_id": "{{ archive_differ_bucket_user_access_key_id }}",
+      "aws_secret_access_key": "{{ archive_differ_bucket_user_secret_access_key }}"
     },
-    "bucket_name": "delphi-covidcast-indicator-output",
+    "bucket_name": "{{ archive_differ_bucket_name }}",
     "daily_cache_dir": "./daily_cache",
     "weekly_cache_dir": "./cache"
   }

ansible/templates/quidel_covidtest-params-prod.json.j2

@@ -54,10 +54,10 @@
   },
   "archive": {
     "aws_credentials": {
-      "aws_access_key_id": "{{ delphi_aws_access_key_id }}",
-      "aws_secret_access_key": "{{ delphi_aws_secret_access_key }}"
+      "aws_access_key_id": "{{ archive_differ_bucket_user_access_key_id }}",
+      "aws_secret_access_key": "{{ archive_differ_bucket_user_secret_access_key }}"
     },
-    "bucket_name": "delphi-covidcast-indicator-output",
+    "bucket_name": "{{ archive_differ_bucket_name }}",
     "cache_dir": "./archivediffer_cache",
     "indicator_prefix": "quidel"
   },

ansible/vars.yaml

@@ -28,7 +28,7 @@ safegraph_aws_secret_access_key: "{{ vault_safegraph_aws_secret_access_key }}"
 # Quidel
 quidel_aws_access_key_id: "{{ vault_quidel_aws_access_key_id }}"
 quidel_aws_secret_access_key: "{{ vault_quidel_aws_secret_access_key }}"
-quidel_aws_bucket_name: 'delphi-quidel-data'
+quidel_aws_bucket_name: "delphi-quidel-data"
 
 # Change Healthcare
 changehc_sftp_host: "{{ vault_changehc_sftp_host }}"
@@ -51,7 +51,10 @@ doctor_visits_midas_user: "{{ vault_doctor_visits_midas_user }}"
 doctor_visits_midas_password: "{{ vault_doctor_visits_midas_password }}"
 
 # NCHS
-nchs_mortality_token: "{{ vault_nchs_mortality_token }}"
+nchs_mortality_token: "{{ vault_cdc_socrata_token }}"
+
+# NWSS
+nwss_wastewater_token: "{{ vault_cdc_socrata_token }}"
 
 # SirCAL
 sir_complainsalot_api_key: "{{ vault_sir_complainsalot_api_key }}"
@@ -90,3 +93,8 @@ grb_wlssecret: "{{ vault_grb_wlssecret }}"
 backfillcorr_aws_access_key_id: "{{ vault_backfillcorr_aws_access_key_id }}"
 backfillcorr_aws_secret_access_key: "{{ vault_backfillcorr_aws_secret_access_key }}"
 backfillcorr_aws_bucket_name: "{{ vault_backfillcorr_aws_bucket_name }}"
+
+# Archive differ S3 bucket
+archive_differ_bucket_user_access_key_id: "{{ vault_archive_differ_bucket_user_access_key_id }}"
+archive_differ_bucket_user_secret_access_key: "{{ vault_archive_differ_bucket_user_secret_access_key }}"
+archive_differ_bucket_name: 'delphi-covidcast-indicator-output-scsdr'

ansible/vault-decrypt.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Create backup directory and copy vault_plain.yaml if it exists.
+if [ -f vault_plain.yaml ]; then
+    echo "Creating backup of vault_plain.yaml..."
+    mkdir -p vault_backup
+    cp -Rvp vault_plain.yaml \
+    "vault_backup/vault_plain.yaml.backup-$(date -u +%Y-%m-%d_T%H-%M-%S_%Z)"
+fi
+
+# Create a new/overwrite vault_plain.yaml using vault.yaml as the source.
+ansible-vault decrypt --output vault_plain.yaml vault.yaml

ansible/vault-encrypt.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Create backup directory and copy vault.yaml if it exists.
+if [ -f vault.yaml ]; then
+    echo "Creating backup of vault.yaml..."
+    mkdir -p vault_backup
+    cp -Rvp vault.yaml \
+    "vault_backup/vault.yaml.backup-$(date -u +%Y-%m-%d_T%H-%M-%S_%Z)"
+fi
+
+# Create a new/overwrite vault.yaml using vault_plain.yaml as the source.
+ansible-vault encrypt --output vault.yaml vault_plain.yaml