[GitHub Actions] Replace k3s with k3d

.github/ci_config/bin/install-dependencies

@@ -9,31 +9,6 @@ sudo apt install -y openjdk-11-jre-headless curl coreutils
 OS="$(uname | tr '[:upper:]' '[:lower:]')"
 ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')"
 
-if [ -n "${KUBECTL_VERSION}" ]; then
-  KUBECTL_VERSION=${KUBECTL_VERSION}
-else
-  KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
-fi
-curl -LO "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/$OS/$ARCH/kubectl"
-curl -LO "https://dl.k8s.io/$KUBECTL_VERSION/bin/$OS/$ARCH/kubectl.sha256"
-echo "$(cat kubectl.sha256) kubectl" | sha256sum --check
-sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-kubectl version --client --output=yaml
-
-# if [ -n "${KREW_VERSION}" ]; then
-#   KREW_VERSION=${KREW_VERSION}
-# else
-#   KREW_VERSION=$(curl -Ls "https://github.com/kubernetes-sigs/krew/releases" | grep 'href="/kubernetes-sigs/krew/releases/tag/v[0-9]*.[0-9]*.[0-9]\"' | sed -E 's/.*\/kubernetes-sigs\/krew\/releases\/tag\/(v[0-9\.]+)".*/\1/g' | head -1)
-# fi
-# KREW="krew-${OS}_${ARCH}"
-# curl -LO "https://github.com/kubernetes-sigs/krew/releases/download/$KREW_VERSION/$KREW.tar.gz"
-# curl -LO "https://github.com/kubernetes-sigs/krew/releases/download/$KREW_VERSION/$KREW.tar.gz.sha256"
-# echo "$(cat $KREW.tar.gz.sha256) $KREW.tar.gz" | sha256sum --check
-# tar zxvf "${KREW}.tar.gz"
-# ./"${KREW}" install krew
-# export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
-# kubectl krew version
-
 if [ -n "${HELM_VERSION}" ]; then
   HELM_VERSION=${HELM_VERSION}
 else

.github/ci_config/ci-config.yaml

@@ -1,5 +1,5 @@
 atomicInstall: false
-base_timeout: 180
+base_timeout: 360
 
 elasticsearch:
   resources:

.github/ci_config/k3d-config.yaml

@@ -9,7 +9,7 @@ volumes:
       - server:0
 registries:
   create:
-    image: ligfx/k3d-registry-dockerd:v0.5
+    image: ligfx/k3d-registry-dockerd:fix_missing_blobs
     proxy:
       remoteURL: "*"
     volumes:

.github/workflows/matrix_includes.json

@@ -0,0 +1,12 @@
+[
+  {
+    "KUBECTL_VERSION": "v1.31.0",
+    "K3S_VERSION": "v1.31.2+k3s1",
+    "runOnBranch": "always"
+  },
+  {
+    "KUBECTL_VERSION": "v1.30.0",
+    "K3S_VERSION": "v1.30.6+k3s1",
+    "runOnBranch": "main"
+  }
+]

.github/workflows/push.yaml

@@ -1,4 +1,4 @@
-name: Install applications on Kubernets
+name: Install applications on Kubernetes
 
 on:
   pull_request:
@@ -9,14 +9,9 @@ on:
         description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)"
         required: false
         default: false
- 
+
 jobs:
   install:
-    strategy:
-      matrix:
-        K3S_VERSION: ["v1.30.6+k3s1", "v1.31.2+k3s1"]
-        exclude:
-          - K3S_VERSION: ${{ github.base_ref != 'refs/heads/main' && 'v1.30.6+k3s1' }}
     runs-on: self-hosted
     steps:
       - name: Cleanup
@@ -29,6 +24,12 @@ jobs:
 
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - run: pip install -r test/features/requirements.txt
+
       - name: Install dependencies
         env:
           HELM_VERSION: "v3.16.3"
@@ -45,33 +46,77 @@ jobs:
           helmfile_mods+='\ \ \ \ \ \ - ../.github/ci_config/secrets.yaml.gotmpl'
           sed -i "/secrets.yaml/a $helmfile_mods" environments.yaml
           sed -i '/_install: /s/false/true/' etc/production.yaml
+          # Deploy a minimal deployment accessible on 'localhost:80' with monitoring enabled
           sed -i '/enable_logging_monitoring: /s/false/true/' etc/production.yaml
+          sed -i '/dev_deployment: /s/false/true/' etc/production.yaml
+          sed -i '/e2e_test: /s/false/true/' etc/production.yaml
 
-      - name: Run helmfile template
-        env:
-          FIREBASE_ADMINSDK_JSON: ${{ secrets.FIREBASE_ADMINSDK_JSON }}
+      - name: Ensure k3d cache directory exists
         run: |
-          helmfile template
+          mkdir -p $HOME/k3d-containerd
 
-      - name: Reinstall K3S
-        env:
-          INSTALL_K3S_VERSION: ${{ matrix.K3S_VERSION }}
-          K3S_KUBECONFIG_MODE: "644"
+  test-helmfile-template:
+    needs: install
+    runs-on: self-hosted
+    steps:
+    - name: Test helmfile template
+      env:
+        FIREBASE_ADMINSDK_JSON: ${{ secrets.FIREBASE_ADMINSDK_JSON }}
+      run: |
+        helmfile template > /tmp/helmfile.yaml
+
+  delete-old-clusters:
+    runs-on: self-hosted
+    needs: test-helmfile-template
+    steps:
+      - name: Delete old clusters
         run: |
-          /usr/local/bin/k3s-uninstall.sh || true
-          curl -sfL https://get.k3s.io | sh -s - --disable traefik --disable-helm-controller
-          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
+          k3d cluster delete --all || true
+
+  matrix-prep:
+    runs-on: self-hosted
+    needs: delete-old-clusters
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: nelonoel/[email protected]
+      - id: set-matrix
+        uses: JoshuaTheMiller/conditional-build-matrix@main
+        with:
+          # inputFile: '.github/workflows/matrix_includes.json' # Default input file path
+          filter: '[?runOnBranch==`${{ env.BRANCH_NAME }}` || runOnBranch==`always`]'
+          addInclude: true
+
+  test-k8s-versions:
+    needs: matrix-prep
+    runs-on: self-hosted
+    strategy:
+      matrix: ${{ fromJson(needs.matrix-prep.outputs.matrix) }}
+    steps:
+      - uses: nolar/setup-k3d-k3s@v1
+        with:
+          version: ${{ matrix.K3S_VERSION }}
+          k3d-args: --k3s-arg --disable=traefik@server:0 --k3s-arg --disable-helm-controller@server:0 --config .github/ci_config/k3d-config.yaml
+          # Note: k3d adds a 'k3d-' prefix to this name
+          k3d-name: ${{ matrix.KUBECTL_VERSION }}
+
+      # TODO would be better to do this with a command line parameter. I have been unable to get this to work.
+      - name: Set kubeContext
+        run: |
+          sed -i "s/kubeContext: .*/kubeContext: k3d-${{ matrix.KUBECTL_VERSION }}/" etc/production.yaml
 
       - name: Install RADAR-Kubernetes
         env:
           FIREBASE_ADMINSDK_JSON: ${{ secrets.FIREBASE_ADMINSDK_JSON }}
         run: |
           echo "kubeconfig: $KUBECONFIG"
-          kubectl get pods --all-namespaces
-
-          helmfile sync --concurrency 1
+          kubectl get pods --all-namespaces --context k3d-${{ matrix.KUBECTL_VERSION }}
+          helmfile sync
 
-      # Enable tmate debugging of manually-triggered workflows if the input option was provided
+      - name: Run end-to-end tests
+        run: |
+          cd test/features
+          behave -D secrets_file=$GITHUB_WORKSPACE/etc/secrets.yaml -D url=http://localhost:80     # Enable tmate debugging of manually-triggered workflows if the input option was provided
       - name: Manually triggered tmate session
         uses: mxschmitt/action-tmate@v3
         if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

environments.yaml.tmpl

@@ -13,12 +13,14 @@ environments:
     {{ end }}
     {{ if .Values.dev_deployment }}
       - ../mods/disable_tls.yaml
-      - ../mods/disable_monitoring_logging.yaml
       - ../mods/localdev.yaml
       - ../mods/minimal.yaml
       - ../mods/minimal_kafka.yaml.gotmpl
       - ../mods/fast_deploy.yaml
     {{ end }}
+    {{ if .Values.e2e_test }}
+      - ../mods/e2e.yaml
+    {{ end }}
 
 ---
 

etc/base.yaml

@@ -16,9 +16,10 @@ enable_logging_monitoring: false
 # Can be disabled when TLS termination is handled upstream of the on-cluster Nginx reverse proxy.
 enable_tls: true
 
-# Minimal deployment for development (disables TLS, monitoring and logging, sets kafka and minio brokers to 1).
+# Minimal deployment for development (disables TLS, sets kafka and minio brokers to 1 and uses minimal RAM footprint).
 dev_deployment: false
-
+# E2e test deployment (applies very fast application loop timings for tests).
+e2e_test: false
 
 # --------------------------------------------------------- 00-init.yaml ---------------------------------------------------------
 
@@ -382,7 +383,6 @@ realtime_dashboard_timescaledb:
 # Make sure to set:
 #- grafana_metrics_timescaledb._install to true
 #- radar_jdbc_connector_grafana._install to true
-#- ksql_server._install to true
 radar_grafana:
   _install: false
   _chart_version: 8.3.7
@@ -615,6 +615,11 @@ velero:
         config:
           s3Url: https://s3.amazon.com # protocol should be specified
 
+radar_mockserver:
+  _install: false
+  _chart_version: 0.1.0
+  _extra_timeout: 0
+
 nifi:
   _install: false
   _chart_version: 1.2.1
@@ -623,3 +628,4 @@ nifi:
     enabled: false
     url: cp-zookeeper-headless
     port: 2181
+

helmfile.d/10-services.yaml

@@ -439,7 +439,7 @@ releases:
     installed: {{ .Values.radar_appserver._install }}
     timeout: {{ add .Values.base_timeout .Values.radar_appserver._extra_timeout }}
     {{ if .Values.radar_appserver_postgresql._install }}needs: [radar-appserver-postgresql]{{ end }}
-    <<: *logFailedRelease    
+    <<: *logFailedRelease
     values:
       - {{ .Values.radar_appserver | toYaml | indent 8 | trim }}
     set:
@@ -707,7 +707,7 @@ releases:
     installed: {{ .Values.radar_rest_sources_backend._install }}
     timeout: {{ add .Values.base_timeout .Values.radar_rest_sources_backend._extra_timeout }}
     {{ if .Values.postgresql._install }}needs: [postgresql]{{ end }}
-    <<: *logFailedRelease    
+    <<: *logFailedRelease
     values:
       - {{ .Values.radar_rest_sources_backend | toYaml | indent 8 | trim }}
     set:
@@ -765,7 +765,7 @@ releases:
     installed: {{ .Values.radar_gateway._install }}
     timeout: {{ add .Values.base_timeout .Values.radar_gateway._extra_timeout }}
     {{ if .Values.cp_schema_registry._install }}needs: [cp-schema-registry]{{ end }}
-    <<: *logFailedRelease    
+    <<: *logFailedRelease
     values:
       - {{ .Values.radar_gateway | toYaml | indent 8 | trim }}
       - {{ .Values.confluent_cloud | toYaml | indent 8 | trim }}
@@ -806,7 +806,7 @@ releases:
     needs:
       # When catalog server is 'ready', kafka can be used.
       - catalog-server
-    <<: *logFailedRelease    
+    <<: *logFailedRelease
     values:
       - "../etc/radar-backend-stream/values.yaml"
       - {{ .Values.radar_backend_stream | toYaml | indent 8 | trim }}
@@ -819,7 +819,7 @@ releases:
     version: {{ .Values.radar_integration._chart_version }}
     installed: {{ .Values.radar_integration._install }}
     timeout: {{ add .Values.base_timeout .Values.radar_integration._extra_timeout }}
-    <<: *logFailedRelease    
+    <<: *logFailedRelease
     values:
       - {{ .Values.radar_integration | toYaml | indent 8 | trim }}
     set:

helmfile.d/20-test.yaml

@@ -0,0 +1,17 @@
+bases:
+- ../environments.yaml
+
+---
+
+{{ readFile "../etc/helmfile-templates.yaml" }}
+
+releases:
+  - name: mockserver
+    namespace: mockserver
+    chart: radar/radar-mockserver
+    version: {{ .Values.radar_mockserver._chart_version }}
+    installed: {{ .Values.radar_mockserver._install }}
+    timeout: {{ add .Values.base_timeout .Values.radar_mockserver._extra_timeout }}
+    <<: *logFailedRelease
+    values:
+      - {{ .Values.radar_mockserver | toYaml | indent 8 | trim }}

mods/disable_monitoring_logging.yaml

@@ -82,3 +82,10 @@ radar_upload_postgresql:
       enabled: false
     primary:
       sidecars: []
+gateway:
+  serviceMonitor:
+    enabled: false
+ksql_server:
+  prometheus:
+    jmx:
+      enabled: false