Merge remote-tracking branch 'origin/dev' into krivard/covid_hosp-facility-running-time

Author: krivard

.github/workflows/performance-tests.yml

@@ -0,0 +1,97 @@
+name: Performance testing
+
+# Run when a PR comment is created (issues and PRs are considered the same entity in the GitHub API)
+on:
+  issue_comment:
+    types: [created]
+
+# Add some extra perms to comment on a PR
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  run-perftests:
+    # Make sure 1. this is a PR, not an issue 2. it contains "/run performance test" anywhere in the body
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run performance test')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up WireGuard
+        uses: egor-tensin/[email protected]
+        with:
+          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
+          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
+          ips: '${{ secrets.WG_PERF_IPS }}'
+          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
+          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Check out repository
+        uses: actions/checkout@v3
+      # Previous step checks out default branch, so we check out the pull request's branch
+      - name: Switch to PR branch
+        run: |
+          hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
+        run: |
+          cd ..
+          mkdir -p driver/repos/delphi
+          cd driver/repos/delphi
+          git clone https://github.com/cmu-delphi/operations
+          git clone https://github.com/cmu-delphi/utils
+          git clone https://github.com/cmu-delphi/flu-contest
+          git clone https://github.com/cmu-delphi/nowcast
+          cd ../../
+
+          cd ..
+          cp -R delphi-epidata driver/repos/delphi/delphi-epidata
+          cd -
+
+          ln -s repos/delphi/delphi-epidata/dev/local/Makefile
+      - name: Build & run epidata
+        run: |
+          cd ../driver
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
+      - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        run: |
+          cd delphi-admin/load-testing/locust
+          docker build -t locust .
+          export CSV=v4-requests-small.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv
+
+  comment-output:
+    runs-on: ubuntu-latest
+    if: success() || failure() # but not if skipped
+    needs: run-perftests
+    steps:
+      - name: Comment run results
+        # URL that links to run results
+        env:
+          GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        uses: actions/github-script@v5
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: '✅ Performance tests complete! Click here to view results: ${{ env.GITHUB_WORKFLOW_URL }}'
+            })

deploy.json

@@ -25,6 +25,13 @@
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
+    {
+      "type": "move",
+      "src": "src/server/utils",
+      "dst": "[[package]]/server/utils/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
     {
       "type": "move",
       "src": "src/server/endpoints",
@@ -74,11 +81,6 @@
       "dst": "[[package]]/acquisition/cdcp/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/cdcp/cdc_upload.php",
-      "dst": "[[auto_web]]/cdc_upload/index.php",
-      "add-header-comment": true
     },
 
     "// acquisition - ght",
@@ -116,16 +118,6 @@
       "dst": "[[package]]/acquisition/wiki/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/wiki/master.php",
-      "dst": "[[auto_web]]/wiki/master.php",
-      "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/wiki/dashboard.php",
-      "dst": "[[auto_web]]/wiki/index.php",
-      "add-header-comment": true
     },
 
     "// acquisition - flusurv",

dev/docker/web/epidata/Dockerfile

@@ -67,6 +67,11 @@ LOG_DB:=delphi_database_epidata_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
 
+M1=
+ifeq ($(shell uname -smp), Darwin arm64 arm)
+$(info M1 system detected, changing docker platform to linux/amd64.)
+	override M1 =--platform linux/amd64
+endif
 
 .PHONY=web
 web:
@@ -80,11 +85,14 @@ web:
 
 	@# Build the web_epidata image
 	@cd repos/delphi/delphi-epidata;\
-		docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
+		docker build -t delphi_web_epidata\
+			$(M1) \
+			-f ./devops/Dockerfile .;\
 		cd -
 
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
+		$(M1) \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \
@@ -102,10 +110,12 @@ db:
 
 	@# Build the database_epidata image
 	@docker build -t delphi_database_epidata \
+		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
 
 	@# Run the database
 	@docker run --rm -p 127.0.0.1:13306:3306 \
+		$(M1) \
 		--network delphi-net --name delphi_database_epidata \
                 --cap-add=sys_nice \
 		delphi_database_epidata >$(LOG_DB) 2>&1 &
@@ -120,6 +130,7 @@ db:
 .PHONY=py
 py:
 	@docker build -t delphi_web_python \
+		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
 .PHONY=all
@@ -128,6 +139,7 @@ all: db web py
 .PHONY=test
 test:
 	@docker run -i --rm --network delphi-net \
+		$(M1) \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
@@ -137,6 +149,7 @@ test:
 .PHONY=bash
 bash:
 	@docker run -it --rm --network delphi-net \
+		$(M1) \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \

dev/docker/web/epidata/README.md

@@ -116,7 +116,7 @@ above. The base images are built first, followed by the derived
 `epidata`-specific images.
 
 - The [`delphi_web_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/web/epidata/README.md) adds
-  the Epidata API to the `delphi_web` image.
+  the Epidata API to the `delphi_web_epidata` image.
 - The
   [`delphi_database_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/database/epidata/README.md)
   adds user accounts, `epidata` & other appropriate databases, and relevant tables
@@ -125,14 +125,8 @@ above. The base images are built first, followed by the derived
 From the root of your workspace, all of the images can be built as follows:
 
 ```bash
-docker build -t delphi_web \
-  -f repos/delphi/operations/dev/docker/web/Dockerfile .
-
-docker build -t delphi_web_epidata \
-  -f repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
-
-docker build -t delphi_database \
-  -f repos/delphi/operations/dev/docker/database/Dockerfile .
+docker build -t delphi_web_epidata\
+			-f ./devops/Dockerfile .;\
 
 docker build -t delphi_database_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
@@ -394,33 +388,3 @@ The command above maps two local directories into the container:
 - `/repos/delphi/delphi-epidata/src`: Just the source code, which forms the
   container's `delphi.epidata` python package.
 
-### server code
-
-Local web sources (e.g. PHP files) can be bind-mounted into a
-`delphi_web_epidata` container as follows:
-
-```bash
-docker run --rm -p 127.0.0.1:10080:80 \
-  --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src/server/api.php,target=/var/www/html/epidata/api.php,readonly \
-  --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src/server/api_helpers.php,target=/var/www/html/epidata/api_helpers.php,readonly \
-  --network delphi-net --name delphi_web_epidata \
-  delphi_web_epidata
-```
-
-The command above mounts two specific files into the image. It may be tempting
-to bind mount the `src/server` directory rather than specific files, however
-that is currently problematic for a couple of reasons:
-
-1. `server/.htaccess` [from the local repository](https://github.com/cmu-delphi/delphi-epidata/blob/main/src/server/.htaccess) uses
-  the `Header` directive. However, the webserver in the container doesn't have
-  the corresponding module enabled. This causes the server to deny access to
-  the API.
-2. `server/database_config.php`
-  [in the image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/web/epidata/assets/database_config.php) contains
-  database credentials for use in conjunction with the
-  `delphi_database_epidata` container during development. However, the same
-  file from [the local repository](https://github.com/cmu-delphi/delphi-epidata/blob/main/src/server/database_config.php) only
-  contains placeholder values. This prevents communication with the database.
-
-There is currently no benefit to bind-mounting sources into the database
-container because schema changes require restarting the container anyway.

dev/docker/web/epidata/assets/database_config.php

@@ -273,10 +273,8 @@ already built the `delphi_python` image above:
 
 ```bash
 # build web and database images for epidata
-docker build -t delphi_web \
-  -f repos/delphi/operations/dev/docker/web/Dockerfile .
-docker build -t delphi_web_epidata \
-  -f repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
+docker build -t delphi_web_epidata\
+			-f ./devops/Dockerfile .;\
 docker build -t delphi_database_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
 

dev/local/Makefile

@@ -11,6 +11,7 @@
 # first party
 from delphi_utils import Nans
 from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
+from delphi.epidata.client.delphi_epidata import Epidata
 
 # use the local instance of the Epidata API
 BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
@@ -22,11 +23,10 @@ def localSetUp(self):
     """Perform per-test setup."""
     self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
-  def request_based_on_row(self, row: CovidcastTestRow, extract_response: Callable = lambda x: x.json(), **kwargs):
+  def request_based_on_row(self, row: CovidcastTestRow, **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    response = extract_response(response)
+    Epidata.BASE_URL = BASE_URL
+    response = Epidata.covidcast(**params) 
 
     return response
 
@@ -155,7 +155,6 @@ def test_csv_format(self):
     # NB 'format' is a Python reserved word
     response = self.request_based_on_row(
       row,
-      extract_response=lambda resp: resp.text,
       **{'format':'csv'}
     )
 
@@ -194,7 +193,7 @@ def test_fields(self):
     row = self._insert_placeholder_set_one()
 
     # limit fields
-    response = self.request_based_on_row(row, fields='time_value,geo_value')
+    response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"})
 
     expected = row.as_api_compatibility_row_dict()
     expected_all = {