Release Delphi Epidata 0.4.7

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.6
+current_version = 0.4.7
 commit = False
 tag = False
 

.github/workflows/performance-tests.yml

@@ -0,0 +1,156 @@
+name: Performance testing
+
+# Run when a PR comment is created (issues and PRs are considered the same entity in the GitHub API)
+on:
+  issue_comment:
+    types: [created]
+
+# Add some extra perms to comment on a PR
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  run-perftests:
+    # Make sure 1. this is a PR, not an issue 2. it contains "/run performance test" anywhere in the body
+    if: github.event.issue.pull_request && contains(github.event.comment.body, '/run performance test')
+    runs-on: ubuntu-latest
+    outputs:
+      request_count: ${{ steps.output.outputs.request_count }}
+      failure_count: ${{ steps.output.outputs.failure_count }}
+      med_time: ${{ steps.output.outputs.med_time }}
+      avg_time: ${{ steps.output.outputs.avg_time }}
+      min_time: ${{ steps.output.outputs.min_time }}
+      max_time: ${{ steps.output.outputs.max_time }}
+      requests_per_sec: ${{ steps.output.outputs.requests_per_sec }}
+    steps:
+      - name: Set up WireGuard
+        uses: egor-tensin/[email protected]
+        with:
+          endpoint: '${{ secrets.WG_PERF_ENDPOINT }}'
+          endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}'
+          ips: '${{ secrets.WG_PERF_IPS }}'
+          allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}'
+          private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}'
+      - name: Check out repository
+        uses: actions/checkout@v3
+      # Previous step checks out default branch, so we check out the pull request's branch
+      - name: Switch to PR branch
+        run: |
+          hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main
+        run: |
+          cd ..
+          mkdir -p driver/repos/delphi
+          cd driver/repos/delphi
+          git clone https://github.com/cmu-delphi/operations
+          git clone https://github.com/cmu-delphi/utils
+          git clone https://github.com/cmu-delphi/flu-contest
+          git clone https://github.com/cmu-delphi/nowcast
+          cd ../../
+
+          cd ..
+          cp -R delphi-epidata driver/repos/delphi/delphi-epidata
+          cd -
+
+          ln -s repos/delphi/delphi-epidata/dev/local/Makefile
+      - name: Build & run epidata
+        run: |
+          cd ../driver
+          sudo make web sql="${{ secrets.DB_CONN_STRING }}"
+      - name: Check out delphi-admin
+        uses: actions/checkout@v3
+        with:
+          repository: cmu-delphi/delphi-admin
+          token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+          path: delphi-admin
+      - name: Build & run Locust
+        continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case
+        run: |
+          cd delphi-admin/load-testing/locust
+          docker build -t locust .
+          export CSV=v4-requests-small.csv
+          touch output_stats.csv && chmod 666 output_stats.csv
+          touch output_stats_history.csv && chmod 666 output_stats_history.csv
+          touch output_failures.csv && chmod 666 output_failures.csv
+          touch output_exceptions.csv && chmod 666 output_exceptions.csv
+          docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output
+      - name: Produce output for summary
+        id: output
+        uses: jannekem/run-python-script-action@v1
+        with:
+          script: |
+            import os
+
+            def write_string(name, value):
+              with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'{name}={value}', file=fh)
+
+            def write_float(name, value):
+              write_string(name, "{:.2f}".format(float(value)))
+
+            with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped:
+              final_line = scraped.readlines()[-1].split(",")
+              write_string('request_count', final_line[2])
+              write_string('failure_count', final_line[3])
+              write_float('med_time', final_line[4])
+              write_float('avg_time', final_line[5])
+              write_float('min_time', final_line[6])
+              write_float('max_time', final_line[7])
+              write_float('requests_per_sec', final_line[9])
+
+      - name: Archive results as artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: locust-output
+          path: |
+            delphi-admin/load-testing/locust/output_*.csv
+
+  comment-success:
+    runs-on: ubuntu-latest
+    if: success()
+    needs: run-perftests
+    steps:
+      - name: Comment run results
+        env:
+          GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        uses: actions/github-script@v5
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `✅ Performance tests complete! Result summary:
+              - Total requests: **${{ needs.run-perftests.outputs.request_count }}**
+              - Total failures: **${{ needs.run-perftests.outputs.failure_count }}**
+              - Min response time: **${{ needs.run-perftests.outputs.min_time }} ms**
+              - Max response time: **${{ needs.run-perftests.outputs.max_time }} ms**
+              - Average response time: **${{ needs.run-perftests.outputs.avg_time }} ms**
+              - Median response time: **${{ needs.run-perftests.outputs.med_time }} ms**
+              - Requests per second: **${{ needs.run-perftests.outputs.requests_per_sec }}**
+
+              Click here to view full results: ${{ env.GITHUB_WORKFLOW_URL }}.`
+            })
+
+  comment-failure:
+    runs-on: ubuntu-latest
+    if: failure()
+    needs: run-perftests
+    steps:
+      - name: Comment run results
+        env:
+          GITHUB_WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        uses: actions/github-script@v5
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `❌ Performance tests failed! Click here to view full results: ${{ env.GITHUB_WORKFLOW_URL }}.`
+            })

deploy.json

@@ -25,6 +25,13 @@
       "match": "^.*\\.(py)$",
       "add-header-comment": true
     },
+    {
+      "type": "move",
+      "src": "src/server/utils",
+      "dst": "[[package]]/server/utils/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
     {
       "type": "move",
       "src": "src/server/endpoints",
@@ -40,6 +47,15 @@
       "add-header-comment": true
     },
 
+    "// acquisition - common",
+    {
+      "type": "move",
+      "src": "src/acquisition/common/",
+      "dst": "[[package]]/acquisition/common/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
+
     "// acquisition - fluview",
     {
       "type": "move",
@@ -65,11 +81,6 @@
       "dst": "[[package]]/acquisition/cdcp/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/cdcp/cdc_upload.php",
-      "dst": "[[auto_web]]/cdc_upload/index.php",
-      "add-header-comment": true
     },
 
     "// acquisition - ght",
@@ -107,16 +118,6 @@
       "dst": "[[package]]/acquisition/wiki/",
       "match": "^.*\\.(py)$",
       "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/wiki/master.php",
-      "dst": "[[auto_web]]/wiki/master.php",
-      "add-header-comment": true
-    },{
-      "type": "move",
-      "src": "src/acquisition/wiki/dashboard.php",
-      "dst": "[[auto_web]]/wiki/index.php",
-      "add-header-comment": true
     },
 
     "// acquisition - flusurv",

dev/local/Makefile

@@ -67,6 +67,11 @@ LOG_DB:=delphi_database_epidata_$(NOW).log
 WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
 DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
 
+M1=
+ifeq ($(shell uname -smp), Darwin arm64 arm)
+$(info M1 system detected, changing docker platform to linux/amd64.)
+	override M1 =--platform linux/amd64
+endif
 
 .PHONY=web
 web:
@@ -80,11 +85,14 @@ web:
 
 	@# Build the web_epidata image
 	@cd repos/delphi/delphi-epidata;\
-		docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
+		docker build -t delphi_web_epidata\
+			$(M1) \
+			-f ./devops/Dockerfile .;\
 		cd -
 
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
+		$(M1) \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \
@@ -102,10 +110,12 @@ db:
 
 	@# Build the database_epidata image
 	@docker build -t delphi_database_epidata \
+		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
 
 	@# Run the database
 	@docker run --rm -p 127.0.0.1:13306:3306 \
+		$(M1) \
 		--network delphi-net --name delphi_database_epidata \
                 --cap-add=sys_nice \
 		delphi_database_epidata >$(LOG_DB) 2>&1 &
@@ -120,6 +130,7 @@ db:
 .PHONY=py
 py:
 	@docker build -t delphi_web_python \
+		$(M1) \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
 .PHONY=all
@@ -128,6 +139,7 @@ all: db web py
 .PHONY=test
 test:
 	@docker run -i --rm --network delphi-net \
+		$(M1) \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
@@ -137,6 +149,7 @@ test:
 .PHONY=bash
 bash:
 	@docker run -it --rm --network delphi-net \
+		$(M1) \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.6
+version = 0.4.7
 
 [options]
 packages =

docs/epidata_development.md

@@ -116,7 +116,7 @@ above. The base images are built first, followed by the derived
 `epidata`-specific images.
 
 - The [`delphi_web_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/web/epidata/README.md) adds
-  the Epidata API to the `delphi_web` image.
+  the Epidata API to the `delphi_web_epidata` image.
 - The
   [`delphi_database_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/database/epidata/README.md)
   adds user accounts, `epidata` & other appropriate databases, and relevant tables
@@ -125,14 +125,8 @@ above. The base images are built first, followed by the derived
 From the root of your workspace, all of the images can be built as follows:
 
 ```bash
-docker build -t delphi_web \
-  -f repos/delphi/operations/dev/docker/web/Dockerfile .
-
-docker build -t delphi_web_epidata \
-  -f repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
-
-docker build -t delphi_database \
-  -f repos/delphi/operations/dev/docker/database/Dockerfile .
+docker build -t delphi_web_epidata\
+			-f ./devops/Dockerfile .;\
 
 docker build -t delphi_database_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
@@ -394,33 +388,3 @@ The command above maps two local directories into the container:
 - `/repos/delphi/delphi-epidata/src`: Just the source code, which forms the
   container's `delphi.epidata` python package.
 
-### server code
-
-Local web sources (e.g. PHP files) can be bind-mounted into a
-`delphi_web_epidata` container as follows:
-
-```bash
-docker run --rm -p 127.0.0.1:10080:80 \
-  --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src/server/api.php,target=/var/www/html/epidata/api.php,readonly \
-  --mount type=bind,source="$(pwd)"/repos/delphi/delphi-epidata/src/server/api_helpers.php,target=/var/www/html/epidata/api_helpers.php,readonly \
-  --network delphi-net --name delphi_web_epidata \
-  delphi_web_epidata
-```
-
-The command above mounts two specific files into the image. It may be tempting
-to bind mount the `src/server` directory rather than specific files, however
-that is currently problematic for a couple of reasons:
-
-1. `server/.htaccess` [from the local repository](https://github.com/cmu-delphi/delphi-epidata/blob/main/src/server/.htaccess) uses
-  the `Header` directive. However, the webserver in the container doesn't have
-  the corresponding module enabled. This causes the server to deny access to
-  the API.
-2. `server/database_config.php`
-  [in the image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/web/epidata/assets/database_config.php) contains
-  database credentials for use in conjunction with the
-  `delphi_database_epidata` container during development. However, the same
-  file from [the local repository](https://github.com/cmu-delphi/delphi-epidata/blob/main/src/server/database_config.php) only
-  contains placeholder values. This prevents communication with the database.
-
-There is currently no benefit to bind-mounting sources into the database
-container because schema changes require restarting the container anyway.