Merge staging into main

.ci.env

@@ -0,0 +1,22 @@
+# This file is used to allow CI to start the compose services. It will typcially
+# not need to be modified.
+
+MYSQL_DATABASE=mysql_database
+MYSQL_USER=mysql_user
+MYSQL_PASSWORD=mysql_password
+MYSQL_PORT=3306
+MYSQL_ROOT_PASSWORD=test123!
+MYSQL_HOST=db
+
+ALLOWED_HOSTS='127.0.0.1,localhost'
+CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
+CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
+
+SECRET_KEY='secret_key'
+DEBUG='True'
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.env.example

@@ -1,11 +1,26 @@
-POSTGRES_DB=postgres_db_name
-POSTGRES_USER=postgres_db_user
-POSTGRES_PASSWORD=postgres_db_password
-POSTGRES_PORT=postgres_db_port
+MYSQL_DATABASE=mysql_database
+MYSQL_USER=mysql_user
+MYSQL_PASSWORD=mysql_password
+MYSQL_PORT=mysql_port
+MYSQL_HOST=mysql_host
+MYSQL_ROOT_PASSWORD=mysql_root_password
 
 ALLOWED_HOSTS='127.0.0.1,localhost'
 CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
 CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
 
 SECRET_KEY='secret_key'
 DEBUG='True'
+
+# Sentry
+# Modify the Sentry config here. A DSN string is the minimum. Check the
+# application's Sentry init section and the Sentry documentation for the
+# defaults and options that may be set. If you don't want to use Sentry you can
+# leave this as is.
+#SENTRY_DSN=''
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.github/workflows/build-and-deploy.yaml

@@ -0,0 +1,58 @@
+name: 'Build and deploy application containers'
+on:
+  push:
+jobs:
+  build-tag-push-deploy:
+    runs-on: ubuntu-latest
+    # CI/CD will run on these branches
+    if: >
+      github.ref == 'refs/heads/main' ||
+      github.ref == 'refs/heads/staging' ||
+      github.ref == 'refs/heads/development'
+    strategy:
+      matrix:
+        # Specify the docker-compose services to build images from
+        service: [sdwebapp, sdnginx]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: cmu-delphi-deploy-machine
+          password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+      - name: Create container image tags
+        id: image-tag
+        run: |
+          baseRef="${GITHUB_REF#*/}"
+          baseRef="${baseRef#*/}"
+          case "${baseRef}" in
+          main)
+            image_tag="latest"
+            ;;
+          *)
+            image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name
+            ;;
+          esac
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT
+      - name: Copy env file
+        run: |
+          cp ./.ci.env ./.env
+      - name: Set up docker-compose
+        uses: ndeloof/[email protected]
+      - name: docker-compose build --push
+        run: |
+          docker-compose build --push ${{ matrix.service }}
+        env:
+          TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}"
+          REGISTRY: "ghcr.io/${{ github.repository_owner }}/"
+      - name: docker-compose down
+        run: |
+          docker-compose down
+      - name: Trigger smee.io webhook to pull new container images
+        run: |
+          curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
+               -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
+               -H "Content-Type: application/x-www-form-urlencoded" \
+               -d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}"

.github/workflows/generate-sphynx-docs.yml

@@ -0,0 +1,52 @@
+name: Update Sphinx Documentation
+
+on:
+  pull_request:
+    branches:
+      - develop
+    types: [opened, synchronize, reopened]
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout source branch
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        sudo apt-get update -y
+        sudo apt-get install mysql-client -y
+        sudo apt-get install graphviz graphviz-dev -y
+        pip install pipenv
+        pipenv requirements > requirements.txt
+        pip install -r requirements.txt
+
+
+    - name: Generate documentation
+      run: |
+        cd src/docs
+        make clean
+        make html
+
+    - name: setup git config
+      run: |
+        # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
+        git config user.name "GitHub Actions Bot"
+        git config user.email "<>"
+
+    - name: Commit and push changes
+      run: |
+        git add -A
+        git commit -m "Update Sphinx documentation"
+        git push

.github/workflows/sync-issue-to-jira.yml

@@ -0,0 +1,92 @@
+name: Sync GitHub issues to Jira
+on: [issues]
+
+jobs:
+  sync-issues:
+    name: Sync issues to Jira
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get the repository name
+        id: repo_name
+        run: echo "REPO_NAME=$(echo ${{ github.repository }} | cut -d '/' -f 2)" >> $GITHUB_OUTPUT
+      - name: Sync issues to Jira
+        env:
+          issue_id: ${{ github.event.issue.html_url }}
+          issue_title: ${{ github.event.issue.title }}
+          issue_body: ${{ github.event.issue.body }}
+          issue_author: ${{ github.event.issue.user.login }}
+        run: |
+          # Add your action code here
+          set -eux
+
+          if [ ${{ github.event_name }} != "issues" ]; then
+            echo "This action only supports the issues event"
+            exit 1
+          fi
+          if [ ${{ github.event.issue.pull_request }} ]; then
+            echo "This action does not support pull requests"
+            exit 0
+          fi
+          # Issue creation with label will trigger 2 events and run twice: one create, one labelled.
+          # let just focus on labelling then for creating issues Jira-side.
+          if [ ${{ github.event_name }} == "issues" ] && [ ${{ github.event.action }} == "opened" ]; then
+            echo "Ignoring creation of issues as a label will trigger a second event."
+            exit 0
+          fi
+          # We only operate on labelled issues or issues that are just unlabeled with our desired label which is 'jira'
+          ## check if one label of labels is our jira label
+          toconsider=${{ contains(github.event.issue.labels.*.name, 'jira') }}
+          if [ "${toconsider}" == false ]; then
+            echo "Our desired label not found on issue or not unlabeled, skipping"
+            exit 0
+          fi
+          # And finally, for the "labeled" event, we are only interested if the new added label is our desired one.
+          if [ ${{ github.event.action }} == "labeled" ] && [ ${{ github.event.label.name }} != 'jira' ]; then
+            echo "Not interested in this action, skipping"
+            exit 0
+          fi
+
+          # Convert markdown to JIRA using mistletoe package which is available starting with impish.
+          # Since GH runners only have LTS versions it's safe to only check for focal which doesn't have the package.
+          if [ $(lsb_release -c -s) == "focal" ]; then
+            echo "Converting Markdown to JIRA is only possible starting with Ubuntu 22.04 (jammy). Pushing verbatim content to JIRA..."
+          else
+            TMPDIR=$(mktemp -d)
+            trap 'rm -rf -- "$TMPDIR"' EXIT
+
+            sudo apt install -y python3-mistletoe
+            echo ${issue_body} > $TMPDIR/body.md
+            issue_body=$(PYTHONPATH=/usr/share/doc/python3-mistletoe mistletoe -r examples.jira_renderer.JIRARenderer $TMPDIR/body.md)
+          fi
+
+          issue_description="*Issue created by ${issue_author}}*
+
+          Description: ${issue_body}
+
+          GitHub URL: ${issue_id}"
+          # Choose Jira action based on event type and action
+          action=""
+          if [ ${{ github.event.action }} == "labeled" ]; then
+            action=Create
+          fi
+
+          echo "PUSHING: $issue_id $action $issue_title $issue_description"
+          # Push to Jira as a json data format
+          data=$(jq -n \
+            --arg id "$issue_id" \
+            --arg title "$issue_title" \
+            --arg description "$issue_description" \
+            --arg action "$action" \
+            --arg project "${{ steps.repo_name.outputs.REPO_NAME }}" \
+            '{data: {id: $id, title: $title, description: $description, action: $action, project: $project}}')
+          curl -X POST -H "Content-Type: application/json" -d "${data}" '${{ secrets.JIRA_WEBHOOK }}'
+        shell: bash  
+
+
+
+
+
+
+
+
+

.github/workflows/tests.yaml

@@ -0,0 +1,79 @@
+name: Django Tests
+on: [pull_request, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      MYSQL_ROOT_PASSWORD: mysql_root_password
+      MYSQL_DATABASE: mysql_database
+      MYSQL_USER: mysql_user
+      MYSQL_PASSWORD: mysql_password
+      MYSQL_HOST: 127.0.0.1
+      MYSQL_PORT: 3306
+
+    services:
+      mysql:
+        image: mysql:latest
+        env:
+          MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }}
+          MYSQL_DATABASE: ${{ env.MYSQL_DATABASE }}
+          MYSQL_USER: ${{ env.MYSQL_USER }}
+          MYSQL_PASSWORD: ${{ env.MYSQL_PASSWORD }}
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          sudo apt-get update -y
+          sudo apt-get install mysql-client -y
+          sudo apt-get install graphviz graphviz-dev -y
+          pip install pipenv
+          pipenv requirements > requirements.txt
+          pip install -r requirements.txt
+
+      - name: Make migrations
+        env:
+          MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }}
+          MYSQL_DATABASE: ${{ env.MYSQL_DATABASE }}
+          MYSQL_USER: ${{ env.MYSQL_USER }}
+          MYSQL_PASSWORD: ${{ env.MYSQL_PASSWORD }}
+          MYSQL_HOST: 127.0.0.1
+          MYSQL_PORT: ${{ env.MYSQL_PORT }}
+
+        run: |
+          mysql -u root -p"${{ env.MYSQL_ROOT_PASSWORD }}" -e "GRANT ALL PRIVILEGES ON *.* TO '${{ env.MYSQL_USER }}'@'%';"
+          python3 ./src/manage.py migrate
+
+      - name: Run linters
+        run: |
+          pip install flake8
+          flake8 .
+
+      - name: Run tests
+        env:
+          MYSQL_ROOT_PASSWORD: ${{ env.MYSQL_ROOT_PASSWORD }}
+          MYSQL_DATABASE: ${{ env.MYSQL_DATABASE }}
+          MYSQL_USER: ${{ env.MYSQL_USER }}
+          MYSQL_PASSWORD: ${{ env.MYSQL_PASSWORD }}
+          MYSQL_HOST: ${{ env.MYSQL_HOST }}
+          MYSQL_PORT: ${{ env.MYSQL_PORT }}
+
+        run: |
+          python3 ./src/manage.py test

.gitignore

@@ -1,11 +1,14 @@
 .env
+.secrets
 /.vscode/
 /src/media/
-/src/static/
+/src/staticfiles/
+*log
 *.mo
 *.pyc
 htmlcov/
 .coverage
 .coverage.*
 coverage.xml
 *.cover
+celerybeat-schedule.*