Merge pull request #21 from cmu-delphi/set-up-deployment

Prepares this application for deployment

Author: korlaxxalrok

.ci.env

@@ -0,0 +1,22 @@
+# This file is used to allow CI to start the compose services. It will typcially
+# not need to be modified.
+
+MYSQL_DATABASE=mysql_database
+MYSQL_USER=mysql_user
+MYSQL_PASSWORD=mysql_password
+MYSQL_PORT=3306
+MYSQL_ROOT_PASSWORD=test123!
+MYSQL_HOST=db
+
+ALLOWED_HOSTS='127.0.0.1,localhost'
+CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
+CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
+
+SECRET_KEY='secret_key'
+DEBUG='True'
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.env.example

@@ -11,3 +11,9 @@ CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
 
 SECRET_KEY='secret_key'
 DEBUG='True'
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.github/workflows/build-and-deploy.yaml

@@ -0,0 +1,57 @@
+name: 'Build and deploy application containers'
+on:
+  push:
+jobs:
+  build-tag-push-deploy:
+    runs-on: ubuntu-latest
+    # CI/CD will run on these branches
+    if: >
+      github.ref == 'refs/heads/master' ||
+      github.ref == 'refs/heads/development'
+    strategy:
+      matrix:
+        # Specify the docker-compose services to build images from
+        service: [sdwebapp, sdnginx]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: cmu-delphi-deploy-machine
+          password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+      - name: Create container image tags
+        id: image-tag
+        run: |
+          baseRef="${GITHUB_REF#*/}"
+          baseRef="${baseRef#*/}"
+          case "${baseRef}" in
+          master)
+            image_tag="latest"
+            ;;
+          *)
+            image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name
+            ;;
+          esac
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT
+      - name: Copy env file
+        run: |
+          cp ./.ci.env ./.env
+      - name: Set up docker-compose
+        uses: ndeloof/[email protected]
+      - name: docker-compose build --push
+        run: |
+          docker-compose build --push ${{ matrix.service }}
+        env:
+          TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}"
+          REGISTRY: "ghcr.io/${{ github.repository_owner }}/"
+      - name: docker-compose down
+        run: |
+          docker-compose down
+      - name: Trigger smee.io webhook to pull new container images
+        run: |
+          curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
+               -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
+               -H "Content-Type: application/x-www-form-urlencoded" \
+               -d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}"

Dockerfile

@@ -12,11 +12,11 @@ RUN apt-get install python3 -y
 RUN apt-get install python3-pip -y
 RUN python3 -m pip install --upgrade pip
 RUN pip3 install pipenv
+RUN pipenv lock
 RUN pipenv requirements > requirements.txt
 RUN pip3 install -r requirements.txt
 
 WORKDIR /home/python
-# RUN pipenv install --system --deploy
 COPY /src .
 COPY /gunicorn/gunicorn.py .
 ENV PATH="/home/python/.local/bin:${PATH}"

Pipfile

@@ -27,6 +27,7 @@ django-coverage-plugin = "*"
 django-extensions-models = "*"
 mypy = "*"
 django-stubs = "*"
+tzdata = "*"
 
 [dev-packages]
 flake8 = "*"

README.md

@@ -77,6 +77,27 @@ $ docker-compose up
 
 Open `http://localhost:8000` to view it in the browser
 
+### To run via docker and emulate production
+
+Though probably not necessary in most cases, if you want to test/modify/emulate how this will run in production you can:
+
+- In `.env` set:
+```
+DEBUG = 'False'
+```
+- Modify the app container's command in `docker-compose.yaml` to run:
+```
+gunicorn signal_documentation.wsgi:application --bind 0.0.0.0:8000"
+
+*(Essentially you'll replace just the last line of the command, switching out the "runserver" line)
+```
+
+Open `http://localhost` to view it in the browser. In this usage your request will be serviced by Nginx instead of the application directly.
+
+The primary use case for this will be when making changes to the Nginx container image that runs in production and hosts the static file content, or also if making changes to the Gunicorn config.
+
+Changes of this sort should be carefully evaluated as they may require interaction with systems managed by devops folks.
+
 ## [Django admin](https://docs.djangoproject.com/en/4.1/ref/contrib/admin/) web interface (user should be `is_staff` or `is_superuser`)
 `http://localhost:8000/admin`
 
@@ -126,3 +147,22 @@ Othervice you will receive Errors during import process:
 3. Import `Signal.base` fields with `SignalBaseResource` - [http://localhost:8000/admin/signals/signal/import/](http://localhost:8000/admin/signals/signal/import/)
 ![Import `Signal.base` field](./docs/image-5.png)
 ![Confirm importing `Signal.base` fields](./docs/image-6.png)
+
+## Deployment
+
+This application gets deployed (at a minimum) to two environmetns:
+
+Production - <https://delphi.cmu.edu/{app_name}>
+
+Staging - <https://staging.delphi.cmu.edu/{app_name}>
+
+Each environment is essentially a bunch of different services all governed by `docker-compose`, running across multiple hosts, with various layering of proxies and load balancers.
+
+### Basic workflow
+
+- A PR merged to either `development` or `master` will trigger CI to build container images that are then tagged (based on the branch name and ":latest" respectively) and stored in our GitHub Packages container image repository.
+- CI triggers a webhook that tells the host systems to pull and run new container images and restart any services that have been updated.
+
+### Control of the deployed environment
+
+The environment and secrets used for deployment live in <https://github.com/cmu-delphi/delphi-ansible-web>. Any changes to the environment should be made there and then tested and validated by devops folks.

docker-compose.yaml

@@ -7,7 +7,7 @@ services:
     container_name: signal_documentation-db
     restart: always
     env_file:
-     - ./.env
+      - ./.env
     environment:
       MYSQL_DATABASE: ${MYSQL_DATABASE}
       MYSQL_USER: ${MYSQL_USER}
@@ -18,11 +18,13 @@ services:
     ports:
       - "3306:3306"
 
-  webapp:
+  sdwebapp:
+    image: ${REGISTRY}signal_documentation-webapp${TAG}
     build: .
     env_file:
-     - ./.env
-    container_name: signal_documentation-web
+      - ./.env
+    container_name: signal_documentation-webapp
+    restart: on-failure
     command: sh -c "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput &&
                     python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput &&
                     python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/available_geography.json &&
@@ -44,6 +46,20 @@ services:
     ports:
       - "6379:6379"
 
+  sdnginx:
+    image: ${REGISTRY}signal_documentation-nginx${TAG}
+    build: ./nginx
+    env_file:
+      - ./.env
+    container_name: signal_documentation-nginx
+    restart: on-failure
+    volumes:
+      - ./src/staticfiles:/staticfiles
+    ports:
+      - "80:80"
+    depends_on:
+      - sdwebapp
 volumes:
   mysql:
   webapp:
+  static:

nginx/default.conf.template

@@ -1,16 +1,17 @@
 server {
-	listen 80;
-	server_name _;
+    listen 80;
+    server_name sdnginx;
 
-	location / {
-		proxy_set_header Host $http_host;
-		proxy_set_header X-Real-IP $remote_addr;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_set_header X-Forwarded-Proto $scheme;
-		proxy_pass http://${APP_HOST}:8000;
-	}
+    location /static/ {
+        autoindex on;
+        alias /staticfiles/;
+    }
 
-	location /usr/src/signal_documentation {
-		alias /static/;
-	}
+    location / {
+        proxy_pass http://sdwebapp:8000/;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
 }