Docker: merge requirements.txt and pin versions #1043 (#1046)

* sorted requirements.txt files
* removed duplicated requirements from ./dev/docker/python/requirements.txt
* reduce runs of "pip install" when creating "delphi_web_python" docker image
* renamed requirements.txt to requirements.api.txt
* merge dev/docker/python/requirements.txt with requirements.dev.txt
* deduplicate packages in requirements.api.txt and requirements.dev.txt
* pinned packages in requirements.dev.txt and removed unused

Co-authored-by: Dmitry Shemetov <[email protected]>

Author: melange396

dev/docker/python/Dockerfile

@@ -6,10 +6,6 @@ RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 # specify a working directory inside the container
 WORKDIR /usr/src/app
 
-# install python packages
-COPY repos/delphi/delphi-epidata/dev/docker/python/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
 # copy over all source files
 COPY repos repos
 RUN chmod -R o+r repos/
@@ -18,4 +14,5 @@ RUN chmod -R o+r repos/
 COPY repos/delphi/delphi-epidata/dev/docker/python/setup.sh .
 RUN bash setup.sh
 
-RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt
+# install python packages
+RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.api.txt -r repos/delphi/delphi-epidata/requirements.dev.txt

dev/docker/python/requirements.txt

@@ -8,7 +8,7 @@ LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epida
 # use delphi's timezome
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 
-COPY requirements.txt /app/requirements_also.txt
+COPY requirements.api.txt /app/requirements_also.txt
 RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:
 #   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )

devops/Dockerfile

@@ -1,15 +1,15 @@
+epiweeks==2.1.2
+Flask==2.2.2
 itsdangerous<2.1
 jinja2==3.0.3
-werkzeug==2.2.2
-Flask==2.2.2
-SQLAlchemy==1.4.40
 mysqlclient==2.1.1
-python-dotenv==0.15.0
+newrelic
 orjson==3.4.7
 pandas==1.2.3
+python-dotenv==0.15.0
 scipy==1.6.2
+SQLAlchemy==1.4.40
+structlog==22.1.0
 tenacity==7.0.0
-newrelic
-epiweeks==2.1.2
 typing-extensions
-structlog==22.1.0
+werkzeug==2.2.2

requirements.txt renamed to requirements.api.txt

@@ -1,8 +1,25 @@
-invoke>=1.4.1
+aiohttp==3.8.3
 black>=20.8b1
-sqlalchemy-stubs>=0.3
+bump2version==1.0.1
+covidcast==0.1.5
+delphi_utils==0.3.6
+docker==6.0.1
+dropbox==11.36.0
+freezegun==1.2.2
+invoke>=1.4.1
+lxml==4.9.1
+matplotlib==3.6.2
 mypy>=0.790
-pytest
+mysql-connector==2.2.9
+numpy==1.22.4
+pycountry==22.3.5
+pymysql==1.0.2
+pytest==7.2.0
+pytest-check==1.3.0
+requests==2.28.1
+sas7bdat==2.2.3
+selenium==4.7.2
+sqlalchemy-stubs>=0.3
+structlog==22.1.0
 tenacity==7.0.0
-bump2version
-requests
+xlrd==2.0.1