Fix the k8ssandra-operator ITs - MED-118 (#838)

* MED-118 Moderinize CI, distros we build for, and Python we support

* Fix indentation

* Run DSE ITs with S3 storage backend only

* Pin poetry version in k8s ITs

.github/workflows/ci.yml

@@ -13,17 +13,15 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.9, "3.10", "3.11"]
         include:
-          - python-version: 3.8
-            tox-py: py38
           - python-version: 3.9
             tox-py: py39
           - python-version: "3.10"
             tox-py: py310
           - python-version: "3.11"
             tox-py: py311
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -57,10 +55,10 @@ jobs:
     # Build debian packages
     strategy:
       matrix:
-        suite: [focal]
+        suite: [jammy]
         include:
-          - suite: focal
-            os-version: ubuntu-20.04
+          - suite: jammy
+            os-version: ubuntu-22.04
       fail-fast: false
     runs-on: ${{ matrix.os-version }}
     steps:
@@ -101,35 +99,29 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          python-version: [3.8, 3.9, "3.10", "3.11"]
+          python-version: [3.9, "3.10", "3.11"]
           it-backend: [local, s3, gcs, minio, azure, azure-hierarchical]
           # IBM not included by default due to lite plan quota being easily exceeded
           #it-backend: [local, s3, gcs, minio, ibm, azure]
-          cassandra-version: [2.2.19, 3.11.11, 4.0.0, 'github:apache/trunk']
+          cassandra-version: [2.2.19, 3.11.17, 4.0.15, 'github:apache/trunk']
           java-version: [8.0.252, 11.0.25]
           include:
             # tweak the experimental flag for cassandra versions
             - cassandra-version: 2.2.19
               experimental: false
-            - cassandra-version: 3.11.11
+            - cassandra-version: 3.11.17
               experimental: false
-            - cassandra-version: 4.0.0
+            - cassandra-version: 4.0.15
               experimental: false
             # explicitly include tests against python 3.11 and one version of cassandra
             - python-version: "3.11"
-              cassandra-version: 4.0.0
+              cassandra-version: 4.0.15
               it-backend: gcs
               experimental: false
               java-version: 11.0.25
             # explicitly include tests against python 3.10 and one version of cassandra
             - python-version: "3.10"
-              cassandra-version: 4.0.0
-              it-backend: gcs
-              experimental: false
-              java-version: 11.0.25
-            # explicitly include tests against python 3.8 and one version of cassandra
-            - python-version: 3.8
-              cassandra-version: 4.0.0
+              cassandra-version: 4.0.15
               it-backend: gcs
               experimental: false
               java-version: 11.0.25
@@ -140,30 +132,29 @@ jobs:
             - cassandra-version: 2.2.19
               java-version: 11.0.25
             # no tests for C* 3.11 with java 11
-            - cassandra-version: 3.11.11
+            - cassandra-version: 3.11.17
               java-version: 11.0.25
             # no tests for C* 4.0 with java 8
-            - cassandra-version: 4.0.0
+            - cassandra-version: 4.0.15
               java-version: 8.0.252
-            # fewer tests against cassandra 3.11.11 (exclude all but s3 storage backends)
+            # fewer tests against cassandra 3.11.17 (exclude all but s3 storage backends)
             # we are not doing the local because it would run a scenario with mgmt-api which no longer supports 3.11
-            # but we still want some tests against 3.11.11, so we use s3 for at least some coverage
+            # but we still want some tests against 3.11.17, so we use s3 for at least some coverage
             - it-backend: local
-              cassandra-version: "3.11.11"
+              cassandra-version: "3.11.17"
             - it-backend: gcs
-              cassandra-version: "3.11.11"
+              cassandra-version: "3.11.17"
             - it-backend: minio
-              cassandra-version: "3.11.11"
+              cassandra-version: "3.11.17"
             - it-backend: azure
-              cassandra-version: "3.11.11"
+              cassandra-version: "3.11.17"
             - it-backend: azure-hierarchical
-              cassandra-version: "3.11.11"
+              cassandra-version: "3.11.17"
             # no tests against non-python3.9, except the explicitly allowed combinations
-            - python-version: 3.8
             - python-version: "3.10"
             - python-version: "3.11"
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     services:
       minio:
         image: lazybit/minio
@@ -337,6 +328,10 @@ jobs:
         with:
           username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
           password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
       - name: Set up Poetry
         uses: snok/install-poetry@v1
         with:
@@ -410,7 +405,7 @@ jobs:
   publish-docker-master:
     needs: [debian-build, build, integration-tests, k8ssandra-e2e-tests]
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - name: Set up QEMU
@@ -455,10 +450,10 @@ jobs:
     if: github.event_name == 'release' && github.event.action == 'published'
     strategy:
       matrix:
-        suite: [focal]
+        suite: [jammy]
         include:
-          - suite: focal
-            os-version: ubuntu-20.04
+          - suite: jammy
+            os-version: ubuntu-22.04
       fail-fast: false
     runs-on: ${{ matrix.os-version }}
     steps:
@@ -488,7 +483,7 @@ jobs:
   publish-docker:
     needs: [publish-debian]
     if: github.event_name == 'release' && github.event.action == 'published'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - name: Set up QEMU
@@ -531,13 +526,13 @@ jobs:
     # We can only release if the build above succeeded first
     needs: [publish-docker]
     if: github.event_name == 'release' && github.event.action == 'published'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: 3.8
+        python-version: 3.10
     - name: Set up Poetry
       with:
         version: 1.8.5

README.md

@@ -60,6 +60,6 @@ You can find the Docker images for Cassandra Medusa at [https://hub.docker.com/r
 Dependencies
 ------------
 
-Medusa requires Python 3.8 or newer.
+Medusa requires Python 3.9 or newer.
 
 For information on the packaged dependencies of Medusa for Apache Cassandra® and their licenses, check out our [open source report](https://app.fossa.com/reports/cac72e73-1214-4e6d-8476-76567e08db21).

debian/cassandra-medusa.install

@@ -1 +1 @@
-medusa-example.ini /etc/medusa
+medusa-example.ini /etc/medusa

debian/control

@@ -3,7 +3,7 @@ Section: python
 Priority: extra
 Maintainer: The Last Pickle <[email protected]>
 Build-Depends:
-  debhelper (>= 9),
+  debhelper (>= 12),
   dh-python,
   python3-all,
   python3-all-dev,
@@ -17,6 +17,6 @@ Standards-Version: 3.9.5
 Package: cassandra-medusa
 Architecture: any
 Pre-Depends: dpkg (>= 1.16.1), python3, python3-venv, ${misc:Pre-Depends}
-Depends: ${python:Depends}, ${shlibs:Depends}, ${misc:Depends}
+Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}
 Description: Backup tool for Cassandra databases
  Make immutable backups for Cassandra databases and extract them to an external storage.

debian/rules

@@ -30,7 +30,7 @@ export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
 
 override_dh_virtualenv:
 	dh_virtualenv \
-	--python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv \
+	--python /usr/bin/python3.10 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv \
 	--preinstall=poetry==1.8.5 --preinstall=dh-poetry==0.2.0 --pip-tool=dh-poetry
 
 override_dh_strip:

docs/Configuration.md

@@ -123,7 +123,7 @@ backup_grace_period_in_days = 10
 ;aws_cli_path = <Location of the aws cli binary if not in PATH>
 
 [monitoring]
-;monitoring_provider = <Provider used for sending metrics. Currently either of "ffwd" or "local">
+;monitoring_provider = <Provider used for sending metrics. Currently just "local">
 
 [ssh]
 ;username = <SSH username to use for restoring clusters>

docs/Installation.md

@@ -61,6 +61,7 @@ curl -1sLf \
 In case of problem, read the full instructions on the [cloudsmith.io documentation](https://cloudsmith.io/~thelastpickle/repos/medusa/setup/#formats-deb)
 
 *Note: since Medusa 0.9 we publish releases for Ubuntu `bionic` and `focal` only.*
+*Note: since Medusa 0.23 we switched `focal` for `jammy`.*
 
 2/ Install Medusa :
 

k8s/Dockerfile

@@ -30,7 +30,7 @@ COPY . /build/
 
 # General requirements
 ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user poetry
+RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user poetry==1.8.5
 
 # Build medusa itself so we can add the executables in the final image
 RUN cd /build && poetry build && poetry install

medusa-example.ini

@@ -134,7 +134,7 @@ use_sudo_for_restore = True
 ;read_timeout = 60
 
 [monitoring]
-;monitoring_provider = <Provider used for sending metrics. Currently either of "ffwd" or "local">
+;monitoring_provider = <Provider used for sending metrics. Currently just "local">
 
 [ssh]
 ;username = <SSH username to use for restoring clusters>

medusa/monitoring/__init__.py

@@ -15,13 +15,11 @@
 
 import logging
 
-from medusa.monitoring.ffwd import FfwdMonitoring
 from medusa.monitoring.noop import NoopMonitoring
 from medusa.monitoring.local import LocalMonitoring
 from medusa.monitoring.dogstatsd import DogStatsdMonitoring
 
 
-PROVIDER_FFWD = 'ffwd'
 PROVIDER_DOG_STATSD = 'dog-statsd'
 PROVIDER_NONE = 'None'
 PROVIDER_INMEM = 'local'
@@ -35,10 +33,7 @@ def __init__(self, config):
 
     def _connect_monitoring(self):
 
-        if self._config.monitoring_provider == PROVIDER_FFWD:
-            logging.info('Monitoring provider is ffwd')
-            return FfwdMonitoring(self._config)
-        elif self._config.monitoring_provider == PROVIDER_NONE:
+        if self._config.monitoring_provider == PROVIDER_NONE:
             logging.info('Monitoring provider is noop')
             return NoopMonitoring(self._config)
         elif self._config.monitoring_provider == PROVIDER_DOG_STATSD:

packaging/build-deb.sh

@@ -3,11 +3,11 @@
 set -e
 
 case $1 in
-  ""|bionic|buster|focal)
+  ""|bionic|buster|jammy)
     suites=("${1:-bionic}")
     ;;
   all)
-    suites=(focal bionic buster)
+    suites=(jammy bionic buster)
     ;;
   *)
     echo "Unknown distribution suite - allowed values: 'all', 'bionic', 'buster'"

packaging/docker-build/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ARG EXTRA_PACKAGES="python-dev python-pip"
-ARG BUILD_IMAGE=ubuntu:18.04
+ARG BUILD_IMAGE=ubuntu:22.04
 FROM ${BUILD_IMAGE}
 
 ARG DEBIAN_FRONTEND=noninteractive

packaging/docker-build/docker-compose.yml

@@ -15,27 +15,17 @@
 version: '2.1'
 
 services:
-  cassandra-medusa-builder-focal:
+  cassandra-medusa-builder-jammy:
     build:
       context: ../..
       dockerfile: packaging/docker-build/Dockerfile
       args:
-        - BUILD_IMAGE=ubuntu:20.04
+        - BUILD_IMAGE=ubuntu:22.04
         - EXTRA_PACKAGES=""
     volumes:
       - ../..:/usr/src/app/cassandra-medusa
       - ../../packages:/usr/src/app/packages
       - ./scripts:/usr/src/app/scripts
-  cassandra-medusa-builder-bionic:
-    build:
-      context: ../..
-      dockerfile: packaging/docker-build/Dockerfile
-      args:
-        - BUILD_IMAGE=ubuntu:18.04
-    volumes:
-      - ../..:/usr/src/app/cassandra-medusa
-      - ../../packages:/usr/src/app/packages
-      - ./scripts:/usr/src/app/scripts
   cassandra-medusa-builder-buster:
     build:
       context: ../..