From 6d82286d0f311ea722b6b0ac8c91f20b01166bb0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 11:43:28 -0800 Subject: [PATCH 1/7] System: update delphi_web_python Docker image - merge operations repo delphi_python Dockerfile into delphi_web_python - copy Python requirements file to this directory - copy setup.sh to this directory --- dev/docker/python/Dockerfile | 20 ++++++++++++++++++- dev/docker/python/requirements.txt | 31 ++++++++++++++++++++++++++++++ dev/docker/python/setup.sh | 27 ++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 dev/docker/python/requirements.txt create mode 100644 dev/docker/python/setup.sh diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index 07d2c6d3b..1ebcd3395 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -1,4 +1,22 @@ # start with the `delphi_python` image -FROM delphi_python +FROM python:3.8-buster + +# use delphi's timezome +RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime + +# specify a working directory inside the container +WORKDIR /usr/src/app + +# install python packages +COPY repos/delphi/delphi-epidata/dev/docker/python/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# copy over all source files +COPY repos repos +RUN chmod -R o+r repos/ + +# configure the image to match the delphi server +COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh . +RUN bash setup.sh RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt diff --git a/dev/docker/python/requirements.txt b/dev/docker/python/requirements.txt new file mode 100644 index 000000000..2aa7a6448 --- /dev/null +++ b/dev/docker/python/requirements.txt @@ -0,0 +1,31 @@ +aiohttp +beautifulsoup4 +covidcast +delphi_utils +docker +dropbox +epiweeks +Flask==1.1.2 +freezegun +google-api-python-client +lxml +matplotlib +mysql-connector +mysqlclient==2.0.2 +newrelic +numpy +orjson==3.4.7 +pandas==1.2.3 +pycountry +pymysql +pytest +pytest-check +python-dotenv==0.15.0 +requests +sas7bdat +scikit-learn +scipy==1.6.2 +selenium +SQLAlchemy==1.3.22 +structlog +xlrd diff --git a/dev/docker/python/setup.sh b/dev/docker/python/setup.sh new file mode 100644 index 000000000..80cc8d2a4 --- /dev/null +++ b/dev/docker/python/setup.sh @@ -0,0 +1,27 @@ +# This script sets up the correct directory structure within the `delphi_img` +# docker image. + +# Some notes on package structure: +# - Python package names can't contain hyphens, so hyphens in repo names are +# replaced with underscores in the package hierarchy. (An exception is the +# repo `delphi-epidata`, which is renamed to simply `epidata`.) +# - Repos are organized such that the main code for the package is inside of +# a `src/` directory. When deployed, `src/` is elided. (An exception is the +# legacy `undef-analysis` repo, which has sources at the top-level.) + +# bail if anything fails +set -e + +# create python package `undefx` +mkdir undefx +mv repos/undefx/py3tester/src undefx/py3tester +mv repos/undefx/undef-analysis undefx/undef_analysis + +# create python package `delphi` +mkdir delphi +mv repos/delphi/operations/src delphi/operations +mv repos/delphi/utils/src delphi/utils +mv repos/delphi/github-deploy-repo/src delphi/github_deploy_repo +mv repos/delphi/delphi-epidata/src delphi/epidata +mv repos/delphi/flu-contest/src delphi/flu_contest +mv repos/delphi/nowcast/src delphi/nowcast From 33037dc50348fcd54f427b235e8c7ee56d1e4fad Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:12:59 -0800 Subject: [PATCH 2/7] devtools: remove unused Docker images from Makefile --- dev/local/Makefile | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/dev/local/Makefile b/dev/local/Makefile index cf3834aec..d0004ef86 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -7,10 +7,6 @@ # # Checks for the delphi-net bridge and creates if it doesn't exist. # -# Creates all prereq images (delphi_database, delphi_python) only if they don't -# exist. If you need to rebuild a prereq, you're probably doing something -# complicated, and can figure out the rebuild command on your own. -# # # Commands: # @@ -97,10 +93,6 @@ db: @# Setup virtual network if it doesn't exist @docker network ls | grep delphi-net || docker network create --driver bridge delphi-net - @# Only build prereqs if we need them - @docker images delphi_database | grep delphi || \ - docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile . - @# Build the database_epidata image @docker build -t delphi_database_epidata \ -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . @@ -120,10 +112,6 @@ db: .PHONY=py py: - @# Build the python image - @docker build -t delphi_python \ - -f repos/delphi/operations/dev/docker/python/Dockerfile . - @docker build -t delphi_web_python \ -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . From 9a3c07d5864a3553fbd0fe141a42ec9f8e390cd6 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:13:24 -0800 Subject: [PATCH 3/7] CI: remove unused Docker images from build --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9d447331..5070eae06 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -51,8 +51,6 @@ jobs: - name: Build docker images run: | - docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile . - docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile . docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . cd ./repos/delphi/delphi-epidata From 96153dbc5b21eb84e1391e0bb27372d45f08d37b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 5 Dec 2022 12:16:14 -0800 Subject: [PATCH 4/7] System: remove stale comment from Dockerfile Co-authored-by: melange396 --- dev/docker/python/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index 1ebcd3395..5bbf98432 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -1,4 +1,3 @@ -# start with the `delphi_python` image FROM python:3.8-buster # use delphi's timezome From 8da9b0feb9c793a73b8a1487bba336da4e533565 Mon Sep 17 00:00:00 2001 From: melange396 Date: Mon, 5 Dec 2022 18:35:18 -0500 Subject: [PATCH 5/7] Docker: merge requirements.txt and pin versions #1043 (#1046) * sorted requirements.txt files * removed duplicated requirements from ./dev/docker/python/requirements.txt * reduce runs of "pip install" when creating "delphi_web_python" docker image * renamed requirements.txt to requirements.api.txt * merge dev/docker/python/requirements.txt with requirements.dev.txt * deduplicate packages in requirements.api.txt and requirements.dev.txt * pinned packages in requirements.dev.txt and removed unused Co-authored-by: Dmitry Shemetov --- dev/docker/python/Dockerfile | 7 ++---- dev/docker/python/requirements.txt | 31 ------------------------ devops/Dockerfile | 2 +- requirements.txt => requirements.api.txt | 14 +++++------ requirements.dev.txt | 27 +++++++++++++++++---- 5 files changed, 32 insertions(+), 49 deletions(-) delete mode 100644 dev/docker/python/requirements.txt rename requirements.txt => requirements.api.txt (100%) diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index 5bbf98432..a16c2fd7a 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -6,10 +6,6 @@ RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime # specify a working directory inside the container WORKDIR /usr/src/app -# install python packages -COPY repos/delphi/delphi-epidata/dev/docker/python/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - # copy over all source files COPY repos repos RUN chmod -R o+r repos/ @@ -18,4 +14,5 @@ RUN chmod -R o+r repos/ COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh . RUN bash setup.sh -RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt +# install python packages +RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt diff --git a/dev/docker/python/requirements.txt b/dev/docker/python/requirements.txt deleted file mode 100644 index 2aa7a6448..000000000 --- a/dev/docker/python/requirements.txt +++ /dev/null @@ -1,31 +0,0 @@ -aiohttp -beautifulsoup4 -covidcast -delphi_utils -docker -dropbox -epiweeks -Flask==1.1.2 -freezegun -google-api-python-client -lxml -matplotlib -mysql-connector -mysqlclient==2.0.2 -newrelic -numpy -orjson==3.4.7 -pandas==1.2.3 -pycountry -pymysql -pytest -pytest-check -python-dotenv==0.15.0 -requests -sas7bdat -scikit-learn -scipy==1.6.2 -selenium -SQLAlchemy==1.3.22 -structlog -xlrd diff --git a/devops/Dockerfile b/devops/Dockerfile index 602ba59bd..f77eb91fe 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -8,7 +8,7 @@ LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epida # use delphi's timezome RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime -COPY requirements.txt /app/requirements_also.txt +COPY requirements.api.txt /app/requirements_also.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt # the file /tmp/requirements.txt is created in the parent docker definition. (see: # https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 ) diff --git a/requirements.txt b/requirements.api.txt similarity index 100% rename from requirements.txt rename to requirements.api.txt index 21f87fa1d..d5cc0e63b 100644 --- a/requirements.txt +++ b/requirements.api.txt @@ -1,15 +1,15 @@ +epiweeks==2.1.2 +Flask==2.2.2 itsdangerous<2.1 jinja2==3.0.3 -werkzeug==2.2.2 -Flask==2.2.2 -SQLAlchemy==1.4.40 mysqlclient==2.1.1 -python-dotenv==0.15.0 +newrelic orjson==3.4.7 pandas==1.2.3 +python-dotenv==0.15.0 scipy==1.6.2 +SQLAlchemy==1.4.40 +structlog==22.1.0 tenacity==7.0.0 -newrelic -epiweeks==2.1.2 typing-extensions -structlog==22.1.0 +werkzeug==2.2.2 diff --git a/requirements.dev.txt b/requirements.dev.txt index 6cf9efeca..88a84acfb 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,8 +1,25 @@ -invoke>=1.4.1 +aiohttp==3.8.3 black>=20.8b1 -sqlalchemy-stubs>=0.3 +bump2version==1.0.1 +covidcast==0.1.5 +delphi_utils==0.3.6 +docker==6.0.1 +dropbox==11.36.0 +freezegun==1.2.2 +invoke>=1.4.1 +lxml==4.9.1 +matplotlib==3.6.2 mypy>=0.790 -pytest +mysql-connector==2.2.9 +numpy==1.22.4 +pycountry==22.3.5 +pymysql==1.0.2 +pytest==7.2.0 +pytest-check==1.3.0 +requests==2.28.1 +sas7bdat==2.2.3 +selenium==4.7.2 +sqlalchemy-stubs>=0.3 +structlog==22.1.0 tenacity==7.0.0 -bump2version -requests +xlrd==2.0.1 From 6a8c7a22152b522d0c13b8d9a00a75ea1bfd1f97 Mon Sep 17 00:00:00 2001 From: george haff Date: Tue, 6 Dec 2022 12:31:39 -0500 Subject: [PATCH 6/7] delphi_web_python docker image build optimizations --- dev/docker/python/Dockerfile | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/dev/docker/python/Dockerfile b/dev/docker/python/Dockerfile index a16c2fd7a..ffce16b0f 100644 --- a/dev/docker/python/Dockerfile +++ b/dev/docker/python/Dockerfile @@ -1,18 +1,11 @@ FROM python:3.8-buster -# use delphi's timezome -RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime - -# specify a working directory inside the container WORKDIR /usr/src/app -# copy over all source files COPY repos repos -RUN chmod -R o+r repos/ - -# configure the image to match the delphi server COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh . -RUN bash setup.sh -# install python packages -RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt +RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime && \ + chmod -R o+r repos/ && \ + bash setup.sh && \ + pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt From 751a84e565a62b8d4ae054c14c98def08256a2a7 Mon Sep 17 00:00:00 2001 From: george haff Date: Tue, 6 Dec 2022 12:55:27 -0500 Subject: [PATCH 7/7] delphi_web_epidata docker image build optimizations --- devops/Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/devops/Dockerfile b/devops/Dockerfile index f77eb91fe..158fb9589 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -5,11 +5,19 @@ RUN npm ci && npm run build FROM tiangolo/meinheld-gunicorn:python3.8 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata -# use delphi's timezome -RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime + +COPY ./devops/gunicorn_conf.py /app +COPY ./devops/start_wrapper.sh / +COPY ./src/server/ /app/app/ +COPY --from=builder ./src/build/lib/ /app/app/lib/ COPY requirements.api.txt /app/requirements_also.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt + +RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \ + && rm -rf /app/app/__pycache__ /app/app/*.php \ + && chmod -R o+r /app/app \ + && chmod 755 /start_wrapper.sh \ + && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt # the file /tmp/requirements.txt is created in the parent docker definition. (see: # https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 ) # this combined requirements installation ensures all version constrants are accounted for. @@ -17,13 +25,5 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt # disable python stdout buffering ENV PYTHONUNBUFFERED 1 -COPY ./devops/gunicorn_conf.py /app -COPY ./devops/start_wrapper.sh / -COPY ./src/server/ /app/app/ -COPY --from=builder ./src/build/lib/ /app/app/lib/ -RUN rm -rf /app/app/__pycache__ /app/app/*.php \ - && chmod -R o+r /app/app \ - && chmod 755 /start_wrapper.sh - ENTRYPOINT [ "/entrypoint.sh" ] CMD [ "/start_wrapper.sh" ]