Merge delphi_web_python Docker image from operations repo (#1043)

dshemetov · melange396 · web-flow · commit 97090cae367d · 2022-12-06T11:52:43.000-08:00
- remove delphi_python Docker image - merge operations repo delphi_python Dockerfile into delphi_web_python - copy Python requirements file to this directory and merge with requirements.dev.txt - copy setup.sh to this directory - renamed requirements.txt to requirements.api.txt - remove unused Docker commands from CI - merge requirements files and pin versions and sort and remove unused packages (#1046) Co-authored-by: melange396 <george.haff@gmail.com> Co-authored-by: Dmitry Shemetov <dshemetov@ucdavis.edu>
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -51,7 +51,6 @@ jobs:
 
       - name: Build docker images
         run: |
-          docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile .
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
           cd ./repos/delphi/delphi-epidata
diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile
@@ -1,4 +1,11 @@
-# start with the `delphi_python` image, which is built from repos/delphi/operations/dev/docker/python/Dockerfile
-FROM delphi_python
+FROM python:3.8-buster
 
-RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt
+WORKDIR /usr/src/app
+
+COPY repos repos
+COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh .
+
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    chmod -R o+r repos/ && \
+    bash setup.sh && \
+    pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt
diff --git a/dev/docker/python/setup.sh b/dev/docker/python/setup.sh
@@ -0,0 +1,27 @@
+# This script sets up the correct directory structure within the `delphi_img`
+# docker image.
+
+# Some notes on package structure:
+#   - Python package names can't contain hyphens, so hyphens in repo names are
+#     replaced with underscores in the package hierarchy. (An exception is the
+#     repo `delphi-epidata`, which is renamed to simply `epidata`.)
+#   - Repos are organized such that the main code for the package is inside of
+#     a `src/` directory. When deployed, `src/` is elided. (An exception is the
+#     legacy `undef-analysis` repo, which has sources at the top-level.)
+
+# bail if anything fails
+set -e
+
+# create python package `undefx`
+mkdir undefx
+mv repos/undefx/py3tester/src undefx/py3tester
+mv repos/undefx/undef-analysis undefx/undef_analysis
+
+# create python package `delphi`
+mkdir delphi
+mv repos/delphi/operations/src delphi/operations
+mv repos/delphi/utils/src delphi/utils
+mv repos/delphi/github-deploy-repo/src delphi/github_deploy_repo
+mv repos/delphi/delphi-epidata/src delphi/epidata
+mv repos/delphi/flu-contest/src delphi/flu_contest
+mv repos/delphi/nowcast/src delphi/nowcast
diff --git a/dev/local/Makefile b/dev/local/Makefile
@@ -7,10 +7,6 @@
 #
 # Checks for the delphi-net bridge and creates if it doesn't exist.
 #
-# Creates all prereq images (delphi_database, delphi_python) only if they don't
-# exist. If you need to rebuild a prereq, you're probably doing something
-# complicated, and can figure out the rebuild command on your own.
-#
 #
 # Commands:
 #
@@ -116,10 +112,6 @@ db:
 
 .PHONY=py
 py:
-	@# Build the python image
-	@docker build -t delphi_python \
-		-f repos/delphi/operations/dev/docker/python/Dockerfile .
-
 	@docker build -t delphi_web_python \
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
diff --git a/devops/Dockerfile b/devops/Dockerfile
@@ -5,25 +5,25 @@ RUN npm ci && npm run build
 
 FROM tiangolo/meinheld-gunicorn:python3.8
 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
-# use delphi's timezome
-RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 
-COPY requirements.txt /app/requirements_also.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
+COPY ./devops/gunicorn_conf.py /app
+COPY ./devops/start_wrapper.sh /
+COPY ./src/server/ /app/app/
+COPY --from=builder ./src/build/lib/ /app/app/lib/
+
+COPY requirements.api.txt /app/requirements_also.txt
+
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
+      && rm -rf /app/app/__pycache__ /app/app/*.php \
+      && chmod -R o+r /app/app \
+      && chmod 755 /start_wrapper.sh \
+      && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:
 #   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
 # this combined requirements installation ensures all version constrants are accounted for.
 
 # disable python stdout buffering
 ENV PYTHONUNBUFFERED 1
 
-COPY ./devops/gunicorn_conf.py /app
-COPY ./devops/start_wrapper.sh /
-COPY ./src/server/ /app/app/
-COPY --from=builder ./src/build/lib/ /app/app/lib/
-RUN rm -rf /app/app/__pycache__ /app/app/*.php \
-      && chmod -R o+r /app/app \
-      && chmod 755 /start_wrapper.sh
-
 ENTRYPOINT [ "/entrypoint.sh" ]
 CMD [ "/start_wrapper.sh" ]
diff --git a/requirements.api.txt b/requirements.api.txt
@@ -1,15 +1,15 @@
+epiweeks==2.1.2
+Flask==2.2.2
 itsdangerous<2.1
 jinja2==3.0.3
-werkzeug==2.2.2
-Flask==2.2.2
-SQLAlchemy==1.4.40
 mysqlclient==2.1.1
-python-dotenv==0.15.0
+newrelic
 orjson==3.4.7
 pandas==1.2.3
+python-dotenv==0.15.0
 scipy==1.6.2
+SQLAlchemy==1.4.40
+structlog==22.1.0
 tenacity==7.0.0
-newrelic
-epiweeks==2.1.2
 typing-extensions
-structlog==22.1.0
+werkzeug==2.2.2
diff --git a/requirements.dev.txt b/requirements.dev.txt
@@ -1,8 +1,25 @@
-invoke>=1.4.1
+aiohttp==3.8.3
 black>=20.8b1
-sqlalchemy-stubs>=0.3
+bump2version==1.0.1
+covidcast==0.1.5
+delphi_utils==0.3.6
+docker==6.0.1
+dropbox==11.36.0
+freezegun==1.2.2
+invoke>=1.4.1
+lxml==4.9.1
+matplotlib==3.6.2
 mypy>=0.790
-pytest
+mysql-connector==2.2.9
+numpy==1.22.4
+pycountry==22.3.5
+pymysql==1.0.2
+pytest==7.2.0
+pytest-check==1.3.0
+requests==2.28.1
+sas7bdat==2.2.3
+selenium==4.7.2
+sqlalchemy-stubs>=0.3
+structlog==22.1.0
 tenacity==7.0.0
-bump2version
-requests
+xlrd==2.0.1

Original file line number	Diff line number	Diff line change
`@@ -7,10 +7,6 @@`
`7`	`7`	`#`
`8`	`8`	`# Checks for the delphi-net bridge and creates if it doesn't exist.`
`9`	`9`	`#`
`10`		`-# Creates all prereq images (delphi_database, delphi_python) only if they don't`
`11`		`-# exist. If you need to rebuild a prereq, you're probably doing something`
`12`		`-# complicated, and can figure out the rebuild command on your own.`
`13`		`-#`
`14`	`10`	`#`
`15`	`11`	`# Commands:`
`16`	`12`	`#`
`@@ -116,10 +112,6 @@ db:`
`116`	`112`
`117`	`113`	`.PHONY=py`
`118`	`114`	`py:`
`119`		`- @# Build the python image`
`120`		`- @docker build -t delphi_python \`
`121`		`- -f repos/delphi/operations/dev/docker/python/Dockerfile .`
`122`		`-`
`123`	`115`	`@docker build -t delphi_web_python \`
`124`	`116`	`-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .`
`125`	`117`