Skip to content

Commit c6ca50e

Browse files
dshemetovdshemetov
committed
feat: add Docker compose, use in Makefile, try in CI
1 parent 52403e6 commit c6ca50e

File tree

4 files changed

+127
-149
lines changed

4 files changed

+127
-149
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 9 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ jobs:
55
build:
66
runs-on: ubuntu-latest
77
steps:
8+
# This imitates the dev/docker/local/install.sh script.
89
- run: mkdir -p repos/undefx
910
- name: Checkout undefx/py3tester
1011
uses: actions/checkout@v2
@@ -18,7 +19,6 @@ jobs:
1819
path: repos/undefx/undef-analysis
1920

2021
- run: mkdir -p repos/delphi
21-
2222
- name: Checkoutcmu-delphi/operations
2323
uses: actions/checkout@v2
2424
with:
@@ -49,47 +49,20 @@ jobs:
4949
repository: cmu-delphi/nowcast
5050
path: repos/delphi/nowcast
5151

52-
- name: Build docker images
53-
run: |
54-
docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
55-
docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
56-
sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
57-
cd ./repos/delphi/delphi-epidata
58-
docker build -t delphi_web_epidata -f ./devops/Dockerfile .
59-
cd ../../../
60-
61-
# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
62-
# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
63-
- name: Start database and Redis services
64-
run: |
65-
docker network create --driver bridge delphi-net
66-
docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
67-
docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
68-
69-
7052
- run: |
71-
wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for
72-
chmod +x wait-for
73-
./wait-for localhost:13306 -- echo 'ready'
74-
sleep 10s
75-
76-
- name: Start delphi_web_epidata
77-
run: |
78-
docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata
79-
docker ps
80-
81-
- name: Run Unit Tests
82-
run: |
83-
docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests
53+
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
54+
ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml
55+
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
56+
ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/
57+
ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/
8458
85-
- name: Run Integration Tests
59+
- name: Run tests
8660
run: |
87-
docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations
61+
make test
8862
8963
- name: Clean Up
9064
run: |
91-
docker stop delphi_database_epidata delphi_web_epidata delphi_redis
92-
docker network remove delphi-net
65+
docker compose down
9366
9467
build_js_client:
9568
runs-on: ubuntu-latest

‎dev/local/Makefile‎

Lines changed: 24 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
#
2626
# all: Runs the commands 'web' 'db' and 'python'.
2727
#
28-
# test: Runs test and integrations in delphi-epidata. If test
28+
# test: Runs test and integrations in delphi-epidata. If test
2929
# optional arg is provided, then only the tests in that subdir
3030
# are run.
3131
#
@@ -65,140 +65,56 @@ NOW:=$(shell date "+%Y-%m-%d")
6565
LOG_WEB:=delphi_web_epidata_$(NOW).log
6666
LOG_DB:=delphi_database_epidata_$(NOW).log
6767
LOG_REDIS:=delphi_redis_instance_$(NOW).log
68-
WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
69-
DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
70-
REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis')
7168

7269
M1=
7370
ifeq ($(shell uname -smp), Darwin arm64 arm)
7471
$(info M1 system detected, changing docker platform to linux/amd64.)
7572
override M1 =--platform linux/amd64
7673
endif
7774

78-
.PHONY=web
79-
web:
80-
@# Stop container if running
81-
@if [ $(WEB_CONTAINER_ID) ]; then\
82-
docker stop $(WEB_CONTAINER_ID);\
83-
fi
84-
85-
@# Setup virtual network if it doesn't exist
86-
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
87-
88-
@# Build the web_epidata image
89-
@cd repos/delphi/delphi-epidata;\
90-
docker build -t delphi_web_epidata\
91-
$(M1) \
92-
-f ./devops/Dockerfile .;\
93-
cd -
94-
95-
@# Run the web server
96-
@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
97-
@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
98-
@docker run --rm -p 127.0.0.1:10080:80 \
99-
$(M1) \
100-
--env "MODULE_NAME=delphi.epidata.server.main" \
101-
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
102-
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
103-
--env "REDIS_HOST=delphi_redis" \
104-
--env "REDIS_PASSWORD=1234" \
105-
--env "API_KEY_ADMIN_PASSWORD=test_admin_password" \
106-
--env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \
107-
--env "TESTING_MODE=True" \
108-
--network delphi-net --name delphi_web_epidata \
109-
delphi_web_epidata >$(LOG_WEB) 2>&1 &
110-
11175
.PHONY=db
11276
db:
113-
@# Stop container if running
114-
@if [ $(DATABASE_CONTAINER_ID) ]; then\
115-
docker stop $(DATABASE_CONTAINER_ID);\
116-
fi
117-
118-
@# Setup virtual network if it doesn't exist
119-
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
120-
121-
@# Build the database_epidata image
122-
@docker build -t delphi_database_epidata \
123-
$(M1) \
124-
-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
125-
126-
@# Run the database
127-
@docker run --rm -p 127.0.0.1:13306:3306 \
128-
$(M1) \
129-
--network delphi-net --name delphi_database_epidata \
130-
--cap-add=sys_nice \
131-
delphi_database_epidata >$(LOG_DB) 2>&1 &
132-
133-
@# Block until DB is ready
134-
@while true; do \
135-
sed -n '/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
136-
tail -1 $(LOG_DB); \
137-
sleep 1; \
138-
done
139-
140-
.PHONY=py
141-
py:
142-
@docker build -t delphi_web_python \
143-
$(M1) \
144-
-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
77+
@docker compose up delphi_database_epidata $(M1) --detach
78+
@docker logs -f delphi_database_epidata >$(LOG_DB) 2>&1 &
79+
80+
.PHONY=web
81+
web:
82+
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose up delphi_web_epidata $(M1) --detach
83+
@docker logs -f delphi_web_epidata >$(LOG_WEB) 2>&1 &
14584

14685
.PHONY=redis
14786
redis:
148-
@# Stop container if running
149-
@if [ $(REDIS_CONTAINER_ID) ]; then\
150-
docker stop $(REDIS_CONTAINER_ID);\
151-
fi
152-
153-
@docker build -t delphi_redis \
154-
$(M1) \
155-
-f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile .
156-
157-
@docker run --rm -d -p 127.0.0.1:6379:6379 \
158-
$(M1) \
159-
--network delphi-net \
160-
--env "REDIS_PASSWORD=1234" \
161-
--name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 &
162-
163-
.PHONY=all
164-
all: db web py redis
87+
@docker compose up delphi_redis $(M1) --detach
88+
@docker logs -f delphi_redis >$(LOG_REDIS) 2>&1 &
16589

16690
.PHONY=test
16791
test:
168-
@docker run -i --rm --network delphi-net \
169-
$(M1) \
170-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
171-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
172-
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
173-
--env "FLASK_SECRET=abc" \
174-
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
92+
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \
93+
python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
17594

17695
.PHONY=r-test
17796
r-test:
178-
@docker run -i --rm --network delphi-net \
179-
$(M1) \
180-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
181-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
182-
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
183-
--env "FLASK_SECRET=abc" \
184-
delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log
185-
97+
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \
98+
Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log
18699

187100
.PHONY=bash
188101
bash:
189-
@docker run -it --rm --network delphi-net \
190-
$(M1) \
191-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
192-
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
193-
--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
194-
--env "FLASK_SECRET=abc" \
195-
delphi_web_python bash
102+
@SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) bash
196103

197104
.PHONY=sql
198105
sql:
199106
@docker run --rm -it --network delphi-net --cap-add=sys_nice \
200107
percona mysql --user=user --password=pass --port 3306 --host delphi_database_epidata epidata
201108

109+
.PHONY=down
110+
down:
111+
@docker compose down
112+
113+
.PHONY=down-force-rebuild
114+
down-force-rebuild:
115+
@docker compose down --rmi all --volumes
116+
@docker compose up --build --force-recreate --detach
117+
202118
.PHONY=clean
203119
clean:
204120
@docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1

‎dev/local/docker-compose.yaml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
networks:
2+
delphi-net:
3+
driver: bridge
4+
5+
services:
6+
delphi_database_epidata:
7+
container_name: delphi_database_epidata
8+
build:
9+
context: .
10+
dockerfile: repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile
11+
cap_add:
12+
- SYS_NICE
13+
ports:
14+
- "127.0.0.1:13306:3306"
15+
networks:
16+
- delphi-net
17+
healthcheck:
18+
test: "mysql --user=user --password=pass --execute \"SHOW DATABASES;\""
19+
start_period: 5s
20+
timeout: 10s
21+
retries: 5
22+
23+
delphi_web_epidata:
24+
container_name: delphi_web_epidata
25+
build:
26+
context: repos/delphi/delphi-epidata
27+
dockerfile: ./devops/Dockerfile
28+
environment:
29+
MODULE_NAME: delphi.epidata.server.main
30+
SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata}
31+
FLASK_SECRET: abc
32+
FLASK_PREFIX: /epidata
33+
LOG_DEBUG: "1"
34+
REDIS_HOST: delphi_redis
35+
REDIS_PASSWORD: 1234
36+
API_KEY_ADMIN_PASSWORD: test_admin_password
37+
API_KEY_REGISTER_WEBHOOK_TOKEN: abc
38+
TESTING_MODE: True
39+
ports:
40+
- "127.0.0.1:10080:80"
41+
networks:
42+
- delphi-net
43+
healthcheck:
44+
test: "curl -f http://localhost/epidata/covidcast || exit 1"
45+
timeout: 10s
46+
retries: 5
47+
48+
delphi_redis:
49+
container_name: delphi_redis
50+
build:
51+
context: .
52+
dockerfile: repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile
53+
environment:
54+
REDIS_PASSWORD: 1234
55+
ports:
56+
- "127.0.0.1:6379:6379"
57+
networks:
58+
- delphi-net
59+
depends_on:
60+
delphi_database_epidata:
61+
condition: service_healthy
62+
delphi_web_epidata:
63+
condition: service_healthy
64+
65+
delphi_web_python:
66+
container_name: delphi_web_python
67+
build:
68+
context: .
69+
dockerfile: repos/delphi/delphi-epidata/dev/docker/python/Dockerfile
70+
environment:
71+
SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata}
72+
FLASK_SECRET: abc
73+
volumes:
74+
- type: bind
75+
source: repos/delphi/delphi-epidata
76+
target: /usr/src/app/delphi/epidata
77+
read_only: true
78+
- type: bind
79+
source: repos/delphi/delphi-epidata/src
80+
target: /usr/src/app/delphi/epidata
81+
read_only: true
82+
networks:
83+
- delphi-net
84+
depends_on:
85+
delphi_database_epidata:
86+
condition: service_healthy
87+
delphi_web_epidata:
88+
condition: service_healthy
89+
delphi_redis:
90+
condition: service_started

‎dev/local/install.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# Bootstrap delphi-epidata development
33
#
4-
# Downloads the repos needed for local delphi-epidata development into current dir
4+
# Downloads the repos needed for local delphi-epidata development into current dir
55
# and provides a Makefile with Docker control commands
66
# as well as pyproject/setup.cfg files for IDE mappings.
77
#
@@ -44,9 +44,8 @@ git clone https://github.com/undefx/py3tester
4444
git clone https://github.com/undefx/undef-analysis
4545
cd ../../
4646

47+
ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml
4748
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
4849
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
49-
cd repos
50-
ln -s delphi/delphi-epidata/dev/local/pyproject.toml
51-
ln -s delphi/delphi-epidata/dev/local/setup.cfg
52-
cd -
50+
ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/
51+
ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/

0 commit comments

Comments
 (0)