V1.3.5 release

.github/workflows/backend.yml

@@ -10,22 +10,17 @@ defaults:
     working-directory: backend
 
 env:
-  DJANGO_SETTINGS_MODULE: metagrid.config.local
-  DOMAIN_NAME: http://localhost:8000
-  DJANGO_SECURE_SSL_REDIRECT: False
-  CORS_ORIGIN_WHITELIST: http://localhost:3000
-  KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
-  KEYCLOAK_REALM: esgf
-  KEYCLOAK_CLIENT_ID: metagrid-localhost
-  DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
-  REACT_APP_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
-  REACT_APP_WGET_API_URL: https://esgf-node.llnl.gov/esg-search/wget
-  REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
-  REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
-  DJANGO_LOGIN_REDIRECT_URL: http://localhost:3000/search
-  DJANGO_LOGOUT_REDIRECT_URL: http://localhost:3000/search
-  GLOBUS_CLIENT_KEY: ${{ secrets.GLOBUS_KEY }}
-  GLOBUS_CLIENT_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  DJANGO_SETTINGS_MODULE: config.settings
+  PGHOST: localhost
+  PGPASSWORD: postgres
+  PGUSER: postgres
+  METAGRID_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
+  METAGRID_WGET_URL: https://esgf-node.llnl.gov/esg-search/wget
+  METAGRID_STATUS_URL: https://esgf-node.llnl.gov/proxy/status
+  METAGRID_SOLR_URL: https://esgf-node.llnl.gov/esg-search
+  METAGRID_SOCIAL_AUTH_GLOBUS_KEY: ${{ secrets.GLOBUS_KEY }}
+  METAGRID_SOCIAL_AUTH_GLOBUS_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  METAGRID_GLOBUS_CLIENT_ID: 21982de0-2b7a-4ba8-bef5-5606ae098201
 
 jobs:
   build:

.github/workflows/containers.yml

@@ -8,6 +8,8 @@ on:
       - 'v*'
 
   pull_request:
+    branches:
+      - '*-release'
 
   workflow_dispatch:
 
@@ -32,14 +34,15 @@ jobs:
         with:
           images: ghcr.io/esgf2-us/metagrid-frontend
           tags: |
-            type=semver,pattern={{version}}
+            type=ref,event=pr
+            type=ref,event=tag
       - uses: docker/build-push-action@v4
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: frontend/
-          file: frontend/docker/production/react/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
+          file: frontend/Dockerfile
+          push: true
           tags: ${{ steps.metadata.outputs.tags }}
           labels: ${{ steps.metadata.outputs.labels }}
   backend:
@@ -60,13 +63,40 @@ jobs:
         with:
           images: ghcr.io/esgf2-us/metagrid-backend
           tags: |
-            type=semver,pattern={{version}}
+            type=ref,event=pr
+            type=ref,event=tag
       - uses: docker/build-push-action@v4
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: backend/
-          file: backend/docker/production/django/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
+          file: backend/Dockerfile
+          push: true
           tags: ${{ steps.metadata.outputs.tags }}
           labels: ${{ steps.metadata.outputs.labels }}
+  test:
+    name: Test Helm Chart
+    runs-on: ubuntu-latest
+    needs: [frontend, backend]
+    if: github.event_name == 'pull_request'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Start minikube
+      uses: medyagh/setup-minikube@latest
+    - name: Set up Helm
+      uses: azure/[email protected]
+    - name: Install Helm Chart
+      working-directory: helm/
+      run: |
+        PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}')
+
+        helm repo add bitnami https://charts.bitnami.com/bitnami
+
+        helm dependency build
+
+        helm install test . \
+          --set frontend.image.tag=pr-${PR_NUMBER} \
+          --set backend.image.tag=pr-${PR_NUMBER} \
+          --wait
+
+        kubectl get pods

.github/workflows/frontend.yml

@@ -12,21 +12,16 @@ defaults:
   run:
     working-directory: frontend
 
-env:
-  REACT_APP_KEYCLOAK_REALM: esgf
-  REACT_APP_KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
-  REACT_APP_KEYCLOAK_CLIENT_ID: metagrid-localhost
-
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
-      - name: Use Node.js 22.x
+      - name: Use Node.js 23.x
         uses: actions/setup-node@v4
         with:
-          node-version: "22.x"
+          node-version: "23.x"
 
       - name: Cache node modules
         uses: actions/cache@v4
@@ -46,13 +41,8 @@ jobs:
       - name: Run Tests
         env:
           RELEASE: dev
-          ENV_FILE: .envs/.react
           HTML_PATH: public
-        run: |
-          # Replaces react-scripts substitution during build for index.html and generates runtime_env.js
-          docker/production/react/entrypoint
-
-          yarn test:coverage
+        run: yarn test:coverage
 
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v3

.github/workflows/helm-chart.yml

@@ -0,0 +1,26 @@
+name: Helm chart
+
+on:
+  push:
+    tags:
+    - 'v*'
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+    - uses: actions/checkout@v4
+    - name: Chart | Push
+      uses: appany/[email protected]
+      with:
+        name: metagrid
+        repository: esgf2-us
+        tag: ${{ github.ref_name }}
+        path: helm
+        registry: ghcr.io
+        registry_username: ${{ github.actor }}
+        registry_password: ${{ secrets.GITHUB_TOKEN }}
+        update_dependencies: 'true'

.github/workflows/pre-commit.yml

@@ -6,17 +6,17 @@ on:
     branches: [master]
 
 jobs:
-  pre-commit-hooks:
+  pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout Code Repository
+      - name: Checkout code
         uses: actions/checkout@v4
 
       # Required to run the local ESLint hook
       - name: Use Node.js 22.x
         uses: actions/setup-node@v4
         with:
-          node-version: "22.x"
+          node-version: '22.x'
 
       - name: Cache node modules
         uses: actions/cache@v4
@@ -36,12 +36,12 @@ jobs:
           yarn install --frozen-lockfile
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: '3.11'
 
-      # Run all pre-commit hooks on all the files.
-      # Getting only staged files can be tricky in case a new PR is opened
-      # since the action is run on a branch in detached head state
-      - name: Install and Run Pre-commit
-        uses: pre-commit/[email protected]
+      - name: Install requirements
+        run: pip install -r backend/requirements/local.txt
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files

.gitignore

@@ -23,7 +23,6 @@ metagrid_configs/backups
 # .nfs files are created when an open file is removed but is still being accessed
 .nfs*
 
-
 ### VisualStudioCode template
 **/.vscode/*
 
@@ -80,8 +79,6 @@ crashlytics.properties
 crashlytics-build.properties
 fabric.properties
 
-
-
 ### Windows template
 # Windows thumbnail cache files
 Thumbs.db
@@ -106,7 +103,6 @@ $RECYCLE.BIN/
 # Windows shortcuts
 *.lnk
 
-
 ### macOS template
 # General
 *.DS_Store
@@ -135,7 +131,6 @@ Network Trash Folder
 Temporary Items
 .apdisk
 
-
 ### SublimeText template
 # Cache files for Sublime Text
 *.tmlanguage.cache
@@ -168,7 +163,6 @@ bh_unicode_properties.cache
 # https://packagecontrol.io/packages/sublime-github
 GitHub.sublime-settings
 
-
 ### Vim template
 # Swap
 [._]*.s[a-v][a-z]
@@ -189,4 +183,11 @@ tags
 .env
 .envs/*
 !.envs/.local/
+docker-compose*-overlay.yml
+
+# Documentation artifacts
+docs/site
 frontend/public/runtime_env.js
+
+#Helm chart deps
+helm/charts/*

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: "docs|node_modules|migrations|.git|.tox"
+exclude: 'docs|node_modules|migrations|.git|.tox'
 default_stages: [commit]
 fail_fast: true
 
@@ -9,27 +9,48 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+        exclude: traefik/*.yml|helm/templates/.*
 
   # Back-end
   # ------------------------------------------------------------------------------
   - repo: https://github.com/pycqa/flake8
     rev: 7.1.1
     hooks:
       - id: flake8
-        args: ["--config=backend/setup.cfg"]
+        args: ['--config=backend/setup.cfg']
         additional_dependencies: [flake8-isort]
 
   - repo: https://github.com/psf/black
     rev: 24.10.0
     hooks:
       - id: black
-        args: ["--config=backend/pyproject.toml"]
+        args: ['--config=backend/pyproject.toml']
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.12.0
     hooks:
       - id: mypy
-        additional_dependencies: ["types-requests"]
+        additional_dependencies: ['types-requests']
+
+  - repo: https://github.com/radeklat/settings-doc
+    rev: '4.3.1'
+    hooks:
+      - id: settings-doc-markdown
+        name: Generate backend settings markdown
+        language: system
+        entry: env PYTHONPATH=$PYTHONPATH:backend settings-doc generate --output-format markdown
+        args:
+          - '--class'
+          - 'config.settings.site_specific.MetagridBackendSettings'
+          - '--update'
+          - 'docs/docs/users/configurable_environment_variables.md'
+          - '--between'
+          - '<!-- start generated backend settings markdown -->'
+          - '<!-- end generated backend settings markdown -->'
+          - '--templates'
+          - 'docs/custom_templates'
+          - '--heading-offset'
+          - '3'
 
   # Front-end
   # ------------------------------------------------------------------------------

.vscode/settings.json

@@ -53,16 +53,18 @@
   "typescript.updateImportsOnFileMove.enabled": "always",
   // ESLint
   // -----------------------------
-  "eslint.validate": [
-    "javascript",
-    "javascriptreact",
-    "typescript",
-    "typescriptreact"
-  ],
+  "eslint.validate": ["javascript", "javascriptreact", "typescript", "typescriptreact"],
   "eslint.workingDirectories": ["./frontend/"],
   // Jest
   // -----------------------------
-  "jest.pathToJest": "yarn test:watch",
+  "jest.rootPath": "./frontend/src",
+  "jest.jestCommandLine": "yarn jest",
+  "jest.monitorLongRun": "off",
+  "jest.enable": true,
+  "jest.runMode": {
+    "type": "on-demand",
+    "coverage": true
+  },
   // HTML, CSS, JSON
   // -----------------------------
   "[html]": {
@@ -78,5 +80,6 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "python.analysis.extraPaths": ["backend/venv/bin/python"],
-  "editor.formatOnSave": true
+  "python.testing.pytestEnabled": true,
+  "python.testing.pytestArgs": ["backend"]
 }

backend/.dockerignore

@@ -1,4 +1,3 @@
 .*
 !.coveragerc
-!.env
 !.pylintrc