Skip to content

Commit 7de6a7d

Browse files
authored
Merge pull request #386 from cmu-delphi/sgratzl/flaskx
Flask based API
2 parents e31ac05 + cbad2cf commit 7de6a7d

File tree

93 files changed

+5810
-140
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+5810
-140
lines changed

.dockerignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/delphi-epidata
2+
/.mypy_cache
3+
/.github
4+
/docs
5+
__pycache__
6+
/node_modules

.env.example

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FLASK_DEBUG=True
2+
SQLALCHEMY_DATABASE_URI=sqlite:///test.db
3+
FLASK_SECRET=abc
4+
SECRET_TWITTER=abc
5+
SECRET_GHT=abc
6+
SECRET_FLUVIEW=abc
7+
SECRET_CDC=abc
8+
SECRET_SENSORS=abc
9+
SECRET_SENSOR_TWTR=abc
10+
SECRET_SENSOR_GFT=abc
11+
SECRET_SENSOR_GHT=abc
12+
SECRET_SENSOR_GHTJ=abc
13+
SECRET_SENSOR_CDC=abc
14+
SECRET_SENSOR_QUID=abc
15+
SECRET_SENSOR_WIKI=abc
16+
SECRET_QUIDEL=abc
17+
SECRET_NOROSTAT=abc
18+
SECRET_AFHSB=abc

.github/workflows/ci.yaml

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,19 @@ jobs:
5151

5252
- name: Build docker images
5353
run: |
54-
docker build -t delphi_web -f repos/delphi/operations/dev/docker/web/Dockerfile .
5554
docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
5655
docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile .
57-
docker build -t delphi_web_epidata -f ./repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
5856
docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
57+
docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
58+
cd ./repos/delphi/delphi-epidata
59+
docker build -t delphi_web_epidata -f ./devops/Dockerfile .
60+
cd ../../../
5961
6062
- name: Start services
6163
run: |
6264
docker network create --driver bridge delphi-net
6365
docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata delphi_database_epidata
64-
docker run --rm -d -p 10080:80 --network delphi-net --name delphi_web_epidata delphi_web_epidata
66+
docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
6567
docker ps
6668
6769
- run: |
@@ -71,13 +73,67 @@ jobs:
7173
sleep 10s
7274
- name: Run Unit Tests
7375
run: |
74-
docker run --rm --network delphi-net delphi_python python3 -m undefx.py3tester.py3tester --use-exit-code repos/delphi/delphi-epidata/tests
76+
docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
7577
7678
- name: Run Integration Tests
7779
run: |
78-
docker run --rm --network delphi-net delphi_python python3 -m undefx.py3tester.py3tester --use-exit-code repos/delphi/delphi-epidata/integrations
80+
docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations
7981
8082
- name: Clean Up
8183
run: |
8284
docker stop delphi_database_epidata delphi_web_epidata
83-
docker network remove delphi-net
85+
docker network remove delphi-net
86+
87+
image:
88+
# only on main and dev branch
89+
if: startsWith(github.ref, 'refs/heads/main') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/sgratzl/flaskx'
90+
runs-on: ubuntu-latest
91+
steps:
92+
- name: Check out code
93+
uses: actions/checkout@v2
94+
- name: Login to GitHub Container Registry
95+
uses: docker/login-action@v1
96+
with:
97+
registry: ghcr.io
98+
username: cmu-delphi-deploy-machine
99+
password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
100+
- name: Build Image
101+
env:
102+
DEVOPS_DOCKER_FILE: ./devops/Dockerfile
103+
run: |
104+
docker build -t repo --file ${DEVOPS_DOCKER_FILE} .
105+
- name: Resolve Tag
106+
id: tagname
107+
run: |
108+
baseRef="${GITHUB_REF#*/}"
109+
imageTag="${baseRef#*/}"
110+
case "${baseRef}" in
111+
main)
112+
imageTag="latest"
113+
;;
114+
*)
115+
imageTag="${baseRef//\//_}" # replace `/` with `_` in branch name
116+
;;
117+
esac
118+
echo "::set-output name=tag::$imageTag"
119+
echo "::set-output name=repo::ghcr.io/${{ github.repository }}"
120+
- name: Push Dev Tag
121+
run: |
122+
docker tag repo ${{ steps.tagname.outputs.repo }}:${{ steps.tagname.outputs.tag }}
123+
docker push ${{ steps.tagname.outputs.repo }}:${{ steps.tagname.outputs.tag }}
124+
# - name: Extract version
125+
# if: startsWith(github.ref, 'refs/heads/main')
126+
# id: extract_version
127+
# run: node -pe "'::set-output name=version::' + require('./package.json').version"
128+
- name: Trigger Webhook
129+
run: |
130+
# trigger a webhook update
131+
curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
132+
-X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
133+
-H "Content-Type: application/x-www-form-urlencoded" \
134+
-d "repository=${{ steps.tagname.outputs.repo }}&tag=${{ steps.tagname.outputs.tag }}"
135+
# - name: Push Version Tag Tag
136+
# if: startsWith(github.ref, 'refs/heads/main')
137+
# run: |
138+
# docker tag repo ${{ steps.tagname.outputs.repo }}:v${{ steps.extract_version.outputs.version }}
139+
# docker push ${{ steps.tagname.outputs.repo }}:v${{ steps.extract_version.outputs.version }}

.gitignore

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
11
__pycache__/
22
.DS_Store
3-
3+
/.vscode
4+
/delphi-epidata
5+
/.env
6+
*.db
7+
/build
8+
/node_modules

build.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
const { process, minimizeJs, rename } = require('delphi-cmu-buildtools');
2+
3+
Promise.all([
4+
process('*.+(coffee|js|py|R)', [], { srcDir: './src/client', dstDir: './build/lib' }),
5+
process('*.js', [minimizeJs(), rename((f) => f.replace('.js', '.min.js'))], { srcDir: './src/client', dstDir: './build/lib' }),
6+
]).then((r) => console.log(r.flat()));

deploy.json

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
"paths": {
55
"root_web": "/var/www/html/epidata",
66
"auto_web": "/home/automation/public_html/public",
7-
"package": "../delphi/epidata"
7+
"package": "../delphi/epidata",
8+
"stasis": "../stasis"
89
},
910
"actions": [
1011

@@ -43,14 +44,28 @@
4344
"add-header-comment": true
4445
},
4546

46-
"// server - API tests",
47+
"// server",
4748
{
4849
"type": "move",
4950
"src": "src/server/",
5051
"dst": "[[package]]/server/",
5152
"match": "^.*\\.(py)$",
5253
"add-header-comment": true
5354
},
55+
{
56+
"type": "move",
57+
"src": "src/server/endpoints",
58+
"dst": "[[package]]/server/endpoints/",
59+
"match": "^.*\\.(py)$",
60+
"add-header-comment": true
61+
},
62+
{
63+
"type": "move",
64+
"src": "src/server/endpoints/covidcast_utils",
65+
"dst": "[[package]]/server/endpoints/covidcast_utils/",
66+
"match": "^.*\\.(py)$",
67+
"add-header-comment": true
68+
},
5469

5570
"// acquisition - fluview",
5671
{
@@ -246,6 +261,22 @@
246261
"add-header-comment": true
247262
},
248263

264+
"// move flask tests out of the way",
265+
{
266+
"type": "move",
267+
"src": "tests/server/",
268+
"dst": "[[stasis]]/tests/",
269+
"match": "^.*\\.(py)$",
270+
"recursive": true
271+
},
272+
{
273+
"type": "move",
274+
"src": "integrations/server/",
275+
"dst": "[[stasis]]/integrations/",
276+
"match": "^.*\\.(py)$",
277+
"recursive": true
278+
},
279+
249280
"// run unit and coverage tests",
250281
{"type": "py3test"}
251282

dev/docker/python/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# start with the `delphi_python` image
2+
FROM delphi_python
3+
4+
RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt

devops/Dockerfile

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM node:lts-buster AS builder
2+
WORKDIR /src
3+
COPY . /src
4+
RUN npm ci && npm run build
5+
6+
FROM tiangolo/meinheld-gunicorn:python3.7
7+
LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
8+
# use delphi's timezome
9+
RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
10+
11+
COPY requirements.txt /app
12+
RUN pip install --no-cache-dir -r requirements.txt
13+
14+
# disable python stdout buffering
15+
ENV PYTHONUNBUFFERED 1
16+
17+
COPY ./devops/gunicorn_conf.py /app
18+
COPY ./devops/start_wrapper.sh /
19+
COPY ./src/server/ /app/app/
20+
COPY --from=builder ./src/build/lib/ /app/app/lib/
21+
RUN rm -rf /app/app/__pycache__ /app/app/*.php \
22+
&& chmod -R o+r /app/app \
23+
&& chmod 755 /start_wrapper.sh
24+
25+
ENTRYPOINT [ "/entrypoint.sh" ]
26+
CMD [ "/start_wrapper.sh" ]

devops/gunicorn_conf.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from __future__ import print_function
2+
3+
import json
4+
import multiprocessing
5+
import os
6+
7+
workers_per_core_str = os.getenv("WORKERS_PER_CORE", "2")
8+
web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
9+
host = os.getenv("HOST", "0.0.0.0")
10+
port = os.getenv("PORT", "80")
11+
bind_env = os.getenv("BIND", None)
12+
use_loglevel = os.getenv("LOG_LEVEL", "info")
13+
if bind_env:
14+
use_bind = bind_env
15+
else:
16+
use_bind = "{host}:{port}".format(host=host, port=port)
17+
18+
cores = multiprocessing.cpu_count()
19+
workers_per_core = float(workers_per_core_str)
20+
default_web_concurrency = workers_per_core * cores
21+
if web_concurrency_str:
22+
web_concurrency = int(web_concurrency_str)
23+
assert web_concurrency > 0
24+
else:
25+
web_concurrency = int(default_web_concurrency)
26+
27+
# Gunicorn config variables
28+
loglevel = use_loglevel
29+
workers = web_concurrency
30+
bind = use_bind
31+
keepalive = 120
32+
33+
errorlog = "-"
34+
accesslog = "-"
35+
capture_output = True
36+
enable_stdio_inheritance = True
37+
38+
# disable limit request line till 414 issue is resolved
39+
limit_request_line = 0
40+
limit_request_field_size = 0
41+
42+
43+
# For debugging and testing
44+
log_data = {
45+
"loglevel": loglevel,
46+
"workers": workers,
47+
"bind": bind,
48+
# Additional, non-gunicorn variables
49+
"workers_per_core": workers_per_core,
50+
"host": host,
51+
"port": port,
52+
}
53+
print(json.dumps(log_data), flush=True)

devops/start_wrapper.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env sh
2+
set -e
3+
4+
# If a New Relic license key is found then we start with custom New Relic
5+
# commands, otherwise we start via start.sh.
6+
if [ -z "${NEW_RELIC_LICENSE_KEY}" ]; then
7+
sh /start.sh
8+
else
9+
newrelic-admin run-program /start.sh
10+
fi

0 commit comments

Comments
 (0)