Merge pull request DefectDojo#9260 from DefectDojo/master-into-bugfix…

…/2.30.0-2.31.0-dev

Release: Merge back 2.30.0 into bugfix from: master-into-bugfix/2.30.0-2.31.0-dev

.github/labeler.yml

@@ -1,39 +1,62 @@
+---
 docs:
-- docs/**/*
-- readme-docs/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - docs/**/*
+      - readme-docs/**/*
 
 docker:
-- docker/**/*
-- docker**
-- Docker*
+  - changed-files:
+    - any-glob-to-any-file:
+      - docker/**/*
+      - docker**
+      - Docker*
 
 helm:
-- helm/defectdojo/*
-- helm/defectdojo/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - helm/defectdojo/*
+      - helm/defectdojo/**/*
 
 "New Migration":
-- dojo/db_migrations/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/db_migrations/*
 
 unittests:
-- unittests/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - unittests/**/*
 
 integration_tests:
-- tests/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - tests/**/*
 
 settings_changes:
-- dojo/settings/settings.dist.py
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/settings/settings.dist.py
 
 apiv2:
-- dojo/api_v2/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/api_v2/**/*
 
 ui:
-  - dojo/static/**/*
-  - dojo/templates/**/*
-  - dojo/templatetags/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/static/**/*
+      - dojo/templates/**/*
+      - dojo/templatetags/**/*
 
 parser:
-  - dojo/tools/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/tools/**/*
 
 localization:
-  - dojo/locale/*
-  - dojo/locale/**/*
+  - changed-files:
+    - any-glob-to-any-file:
+      - dojo/locale/*
+      - dojo/locale/**/*

.github/workflows/fetch-oas.yml

@@ -0,0 +1,50 @@
+name: Fetch OpenAPI Specifications
+
+on:
+  workflow_call:
+
+jobs:
+  oas_fetch:
+    name: Fetch OpenAPI Specifications
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        file-type: [yaml, json]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # load docker images from build jobs
+      - name: Load images from artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Load docker images
+        run: |-
+             docker load -i nginx/nginx-alpine_img
+             docker load -i django/django-alpine_img
+             docker images
+
+      - name: Start Dojo
+        run: docker-compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps -d postgres nginx uwsgi
+        env:
+          DJANGO_VERSION: alpine
+          NGINX_VERSION: alpine
+
+      - name: Download OpenAPI Specifications
+        run: |-
+             wget 'http://localhost:8080/api/v2/oa3/schema/?format=${{ matrix.file-type }}' -O oas.${{ matrix.file-type }} --tries=10 --retry-on-http-error=502
+
+      - name: Logs
+        if: always()
+        run: docker-compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env logs --tail="2500"
+
+      - name: Shutdown
+        if: always()
+        run: docker-compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env down
+
+      - name: Upload oas.${{ matrix.file-type }} as artifact
+        uses:  actions/upload-artifact@v3
+        with:
+          name: oas-${{ matrix.file-type }}
+          path: oas.${{ matrix.file-type }}
+          retention-days: 1

.github/workflows/pr-labeler.yml

@@ -9,9 +9,12 @@ on:
 
 jobs:
   labeler:
+    permissions:
+      contents: read
+      pull-requests: write
     name: "Autolabeler"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v4
-      with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+      - uses: actions/labeler@v5
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release-drafter.yml

@@ -13,14 +13,50 @@ on:
     # branches to consider in the event; optional, defaults to all
     branches:
       - master
-      - dev
 
 jobs:
+  build-docker-containers:
+    uses: ./.github/workflows/build-docker-images-for-testing.yml
+    secrets: inherit
+
+  oas-fetch:
+    needs: build-docker-containers
+    uses: ./.github/workflows/fetch-oas.yml
+    secrets: inherit
+
   update_release_draft:
+    needs: oas-fetch
     runs-on: ubuntu-latest
     steps:
-      - uses: release-drafter/[email protected]
+      - name: Load OAS files from artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Create Release
+        id: create_release
+        uses: release-drafter/[email protected]
         with:
           version: ${{github.event.inputs.version}}  
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload Release Asset - OpenAPI Specification - YAML
+        id: upload-release-asset-yaml
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./oas-yaml/oas.yaml
+          asset_name: oas.yaml
+          asset_content_type: application/vnd.oai.openapi
+
+      - name: Upload Release Asset - OpenAPI Specification - JSON
+        id: upload-release-asset-json
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./oas-json/oas.json
+          asset_name: oas.json
+          asset_content_type: application/json

.github/workflows/release-x-manual-helm-chart.yml

@@ -1,3 +1,4 @@
+---
 name: "release-X: Release helm chart"
 
 env:
@@ -56,6 +57,16 @@ jobs:
              helm dependency list ./helm/defectdojo
              helm dependency update ./helm/defectdojo
       
+      - name: Add yq
+        uses: mikefarah/yq@master
+
+      - name: Pin version docker version
+        id: pin_image
+        run: |-
+          yq --version
+          yq -i '.tag="${{ github.event.inputs.release_number }}"'  helm/defectdojo/values.yaml
+          echo "Current image tag:`yq -r '.tag' helm/defectdojo/values.yaml`"
+
       - name: Package Helm chart
         id: package-helm-chart
         run: |
@@ -83,6 +94,7 @@ jobs:
           git config --global user.email "${{ env.GIT_EMAIL }}"
           git remote update
           git fetch --all
+          git stash
           git checkout helm-charts
           git pull
           if [ ! -f ./index.yaml ]; then

.github/workflows/shellcheck.yml

@@ -0,0 +1,125 @@
+---
+name: Shellcheck
+on:
+  pull_request:
+env:
+  SHELLCHECK_REPO: 'koalaman/shellcheck'
+  SHELLCHECK_VERSION: 'v0.9.0'
+  SHELLCHECK_SHA: '038fd81de6b7e20cc651571362683853670cdc71'
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Grab shellcheck
+        run: |
+          set -e
+
+          SHELLCHECK_TARBALL_URL="https://github.com/${SHELLCHECK_REPO}/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz"
+          SHELLCHECK_TARBALL_LOC="shellcheck.tar.xz"
+          curl -L "${SHELLCHECK_TARBALL_URL}" -o "${SHELLCHECK_TARBALL_LOC}"
+          tarball_sha=$(shasum ${SHELLCHECK_TARBALL_LOC} | awk '{print $1}')
+          if [ "${tarball_sha}" != "${SHELLCHECK_SHA}" ]; then
+            echo "Got invalid SHA for shellcheck: ${tarball_sha}"
+            exit 1
+          fi
+          tar -xvf "${SHELLCHECK_TARBALL_LOC}"
+          cd "shellcheck-${SHELLCHECK_VERSION}" || exit 1
+          mv shellcheck "${GITHUB_WORKSPACE}/shellcheck"
+
+      - name: Run shellcheck
+        shell: bash
+        run: |
+          set -o pipefail
+
+          # Make sure we already put the proper shellcheck binary in place
+          if [ ! -f "./shellcheck" ]; then
+              echo "shellcheck not found"
+              exit 1
+          fi
+
+          # Make sure we know what to compare the PR's changes against
+          if [ -z "${GITHUB_BASE_REF}" ]; then
+              echo "No base reference supplied"
+              exit 1
+          fi
+
+          num_findings=0
+
+          # Execute shellcheck and add errors based on the output
+          run_shellcheck() {
+              local modified_shell_script="${1}"
+              local findings_file="findings.txt"
+
+              # Remove leftover findings file from previous iterations
+              if [ -f "${findings_file}" ]; then
+                  rm "${findings_file}"
+              fi
+
+              echo "Running shellcheck against ${modified_shell_script}..."
+
+              # If shellcheck reported no errors (exited with 0 status code), return
+              if ./shellcheck -f json -S warning "${modified_shell_script}" | jq -c '.[]' > "${findings_file}"; then
+                  return 0
+              fi
+
+              # Walk each of the individual findings
+              while IFS= read -r finding; do
+                  num_findings=$((num_findings+1))
+
+                  line=$(echo "${finding}" | jq '.line')
+                  end_line=$(echo "${finding}" | jq '.endLine')
+                  column=$(echo "${finding}" | jq '.column')
+                  end_column=$(echo "${finding}" | jq '.endColumn')
+                  code=$(echo "${finding}" | jq '.code')
+                  title="SC${code}"
+                  message="$(echo "${finding}" | jq -r '.message') See https://github.com/koalaman/shellcheck/wiki/${title}"
+
+                  echo "Line: ${line}"
+                  echo "End line: ${end_line}"
+                  echo "Column: ${column}"
+                  echo "End column: ${end_column}"
+                  echo "Title: ${title}"
+                  echo "Message: ${message}"
+
+                  # Raise an error with the file/line/etc
+                  echo "::error file=${modified_shell_script},line=${line},endLine=${end_line},column=${column},endColumn=${end_column},title=${title}::${message}"
+              done < ${findings_file}
+          }
+
+          # Find the shell scripts that were created or modified by this PR
+          find_modified_shell_scripts() {
+              shell_scripts="shell_scripts.txt"
+              modified_files="modified_files.txt"
+              modified_shell_scripts="modified_shell_scripts.txt"
+
+              find . -name "*.sh" -or -name "*.bash" | sed 's#^\./##' > "${shell_scripts}"
+              git diff --name-only "origin/${GITHUB_BASE_REF}" HEAD > "${modified_files}"
+
+              if [ ! -s "${shell_scripts}" ] || [ ! -s "${modified_files}" ]; then
+                  echo "No modified shell scripts detected"
+                  exit 0
+              fi
+
+              if ! grep -Fxf "${shell_scripts}" "${modified_files}" > "${modified_shell_scripts}"; then
+                echo "No modified shell scripts detected"
+                exit 0
+              fi
+          }
+
+          git fetch origin "${GITHUB_BASE_REF}" || exit 1
+
+          find_modified_shell_scripts
+
+          # Loop through the modified shell scripts
+          while IFS= read -r modified_shell_script; do
+              run_shellcheck "${modified_shell_script}"
+          done < ${modified_shell_scripts}
+
+          # If shellcheck reported any findings, fail the workflow
+          if [ ${num_findings} -gt 0 ]; then
+              echo "shellcheck reported ${num_findings} findings."
+              exit 1
+          fi

.github/workflows/test-helm-chart.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           version: v3.4.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.7
 

Dockerfile.integration-tests-debian

@@ -1,7 +1,7 @@
 
 # code: language=Dockerfile
 
-FROM openapitools/openapi-generator-cli:v7.1.0@sha256:6e6da58bad0078a5f79f666a8b09e9b512528d92526a0f611dda3963f9672115 as openapitools
+FROM openapitools/openapi-generator-cli:v7.2.0@sha256:9eab779faa2525b1474c4159ec335d913ee3cee00f641552a2305b0a4d7db8f7 as openapitools
 FROM python:3.11.4-slim-bullseye@sha256:40319d0a897896e746edf877783ef39685d44e90e1e6de8d964d0382df0d4952 as build
 WORKDIR /app
 RUN \

Dockerfile.nginx-alpine

@@ -140,7 +140,7 @@ COPY manage.py ./
 COPY dojo/ ./dojo/
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.25.3-alpine@sha256:db353d0f0c479c91bd15e01fc68ed0f33d9c4c52f3415e63332c3d0bf7a4bb77
+FROM nginx:1.25.3-alpine@sha256:a59278fd22a9d411121e190b8cec8aa57b306aa3332459197777583beb728f59
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

Dockerfile.nginx-debian

@@ -75,7 +75,7 @@ COPY dojo/ ./dojo/
 
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.25.3-alpine@sha256:db353d0f0c479c91bd15e01fc68ed0f33d9c4c52f3415e63332c3d0bf7a4bb77
+FROM nginx:1.25.3-alpine@sha256:a59278fd22a9d411121e190b8cec8aa57b306aa3332459197777583beb728f59
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/