Merge pull request #932 from deNBI/dev

dweinholz · web-flow · commit 4b8a82ff1116 · 2026-03-30T09:07:56.000+02:00
Dev
diff --git a/.env.in b/.env.in
@@ -31,4 +31,4 @@ LOG_FILE=log/portal_client.log
 LOG_BACKUP_COUNT=5
 LOG_MAX_BYTES=1073741824
 
-BIBIGRID_TAG=bibigrid-rest-3.0.0
+BIBIGRID_TAG=bibigrid-rest-3.4.5
diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml
@@ -28,15 +28,15 @@ jobs:
         run: echo "The selected tag is ${{ steps.tag.outputs.TAG }}"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
-      - name: Login to Quay.io
+      - name: Login to de.NBI OCI
         uses: docker/login-action@v4
         with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USERNAME }}
-          password: ${{ secrets.QUAY_TOKEN }}
-      - name: Build and publish image to Quay
-        uses: docker/build-push-action@v6
+          registry: oci.bi.denbi.de
+          username: ${{ secrets.OCI_USERNAME }}
+          password: ${{ secrets.OCI_TOKEN }}
+      - name: Build and publish image to de:NBI OCI
+        uses: docker/build-push-action@v7
         with:
           file: Dockerfile
           push: true
-          tags: quay.io/denbicloud/simplevm-client:${{ steps.tag.outputs.TAG }}
+          tags: oci.bi.denbi.de/simplevm/simplevm-client:${{ steps.tag.outputs.TAG }}
diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml
@@ -14,15 +14,15 @@ jobs:
      - uses: actions/checkout@v6.0.2
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
-     - name: Login to Quay.io
+     - name: Login to de.NBI OCI
        uses: docker/login-action@v4
        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USERNAME }}
-          password: ${{ secrets.QUAY_TOKEN }}
-     - name: Build and publish image to Quay
-       uses: docker/build-push-action@v6
+          registry: oci.bi.denbi.de
+          username: ${{ secrets.OCI_USERNAME }}
+          password: ${{ secrets.OCI_TOKEN }}
+     - name: Build and publish image to de.NBI OCI
+       uses: docker/build-push-action@v7
        with:
           file: Dockerfile
           push: true
-          tags: quay.io/denbicloud/simplevm-client:${{ steps.get_version.outputs.VERSION }}
+          tags: oci.bi.denbi.de/simplevm/simplevm-client:${{ steps.get_version.outputs.VERSION }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,7 +11,7 @@ repos:
 
   - repo: https://github.com/psf/black-pre-commit-mirror
 
-    rev: 26.1.0
+    rev: 26.3.1
     hooks:
       - id: black
         language_version: python3.12
diff --git a/docker-compose.bibigrid.host.yml b/docker-compose.bibigrid.host.yml
@@ -1,7 +1,7 @@
 services:
   simplevm-client:
     container_name: simplevm-client
-    image: quay.io/denbicloud/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
+    image: oci.bi.denbi.de/simplevm/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
     expose:
       - "9090"  # Expose the service port
     env_file:
@@ -20,13 +20,13 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
 
   simplevm_bibigrid:
     container_name: simplevm_bibigrid
-    image: quay.io/denbicloud/bibigrid:${BIBIGRID_TAG}
+    image: oci.bi.denbi.de/simplevm/bibigrid:${BIBIGRID_TAG}
     network_mode: host  # Use the host network namespace
 
     volumes:
@@ -42,7 +42,7 @@ services:
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.3.1
+    image: docker.elastic.co/beats/filebeat:9.3.2
     network_mode: host  # Use the host network namespace
 
     env_file:
diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml
@@ -1,7 +1,7 @@
 services:
   simplevm-client:
     container_name: simplevm-client
-    image: quay.io/denbicloud/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
+    image: oci.bi.denbi.de/simplevm/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
     ports:
       - ${CLIENT_PORT}:9090
     env_file:
@@ -20,15 +20,15 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_bibigrid:
     container_name: simplevm_bibigrid
-    image: quay.io/denbicloud/bibigrid:${BIBIGRID_TAG}
+    image: oci.bi.denbi.de/simplevm/bibigrid:${BIBIGRID_TAG}
     volumes:
       - ./scripts/bibigrid/clear_keys_cron:/etc/crontabs/root
     env_file:
@@ -43,7 +43,7 @@ services:
 
   # filebeat
   simplevm_filebeat:
-    image: docker.elastic.co/beats/filebeat:9.3.1
+    image: docker.elastic.co/beats/filebeat:9.3.2
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml
@@ -20,15 +20,15 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_bibigrid:
     container_name: simplevm_bibigrid
-    image: quay.io/denbicloud/bibigrid:${BIBIGRID_TAG}
+    image: oci.bi.denbi.de/simplevm/bibigrid:${BIBIGRID_TAG}
     volumes:
       - ./clouds.yaml:/root/.config/bibigrid/clouds.yaml
     expose:
@@ -38,7 +38,7 @@ services:
 
   simplevm_metadata_server:
     container_name: simplevm_metadata_server
-    image: quay.io/denbicloud/simplevm-metadata-server:1.0.0
+    image: oci.bi.denbi.de/simplevm/simplevm-metadata-server:1.0.0
     ports:
       - "8008:8008"
     depends_on:
@@ -54,7 +54,7 @@ services:
   simplevm_metadata_server_redis:
     container_name: simplevm_metadata_server_redis
 
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.dev.giessen.yml b/docker-compose.dev.giessen.yml
@@ -20,15 +20,15 @@ services:
 
   simplevm_client_giessen_redis:
     container_name: simplevm_client_giessen_redis
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_metadata_server:
     container_name: simplevm_metadata_server
-    image: quay.io/denbicloud/simplevm-metadata-server:1.0.0
+    image: oci.bi.denbi.de/simplevm/simplevm-metadata-server:1.0.0
     ports:
       - "8008:8008"
     depends_on:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -20,15 +20,15 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_metadata_server:
     container_name: simplevm_metadata_server
-    image: quay.io/denbicloud/simplevm-metadata-server:1.0.0
+    image: oci.bi.denbi.de/simplevm/simplevm-metadata-server:1.0.0
     ports:
       - "8008:8008"
     depends_on:
@@ -44,7 +44,7 @@ services:
   simplevm_metadata_server_redis:
     container_name: simplevm_metadata_server_redis
 
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.host.yml b/docker-compose.host.yml
@@ -1,7 +1,7 @@
 services:
   simplevm-client:
     container_name: simplevm-client
-    image: quay.io/denbicloud/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
+    image: oci.bi.denbi.de/simplevm/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
     expose:
       - "9090"  # Expose the service port
     env_file:
@@ -21,15 +21,15 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.3.1
+    image: docker.elastic.co/beats/filebeat:9.3.2
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   simplevm-client:
     container_name: simplevm-client
-    image: quay.io/denbicloud/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
+    image: oci.bi.denbi.de/simplevm/simplevm-client:${CLOUD_PORTAL_CLIENT_TAG}
     ports:
       - ${CLIENT_PORT}:9090
     env_file:
@@ -20,7 +20,7 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.6.1
+    image: redis:8.6.2
     expose:
       - "6379"
     networks:
@@ -29,7 +29,7 @@ services:
   # filebeat
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.3.1
+    image: docker.elastic.co/beats/filebeat:9.3.2
     env_file:
       - .env
     volumes:
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -1,4 +1,4 @@
 pytest==9.0.2
-coverage==7.13.4
-pytest-cov==7.0.0
+coverage==7.13.5
+pytest-cov==7.1.0
 
diff --git a/requirements.txt b/requirements.txt
@@ -1,15 +1,15 @@
-setuptools==82.0.0
+setuptools==82.0.1
 thrift==0.22.0
-openstacksdk==4.9.0
+openstacksdk==4.10.0
 deprecated==1.3.1
 Click==8.3.1
-ansible==12.2.0
+ansible==13.4.0
 flake8==7.3.0
 paramiko==4.0.0
 ruamel.yaml==0.19.1
 pyvim==3.0.3
-redis==7.2.1
-requests==2.32.5
+redis==7.4.0
+requests==2.33.0
 pyyaml==6.0.3
 pre-commit==4.5.1
 types-PyYAML==6.0.12.20250915
diff --git a/simple_vm_client/bibigrid_connector/bibigrid_connector.py b/simple_vm_client/bibigrid_connector/bibigrid_connector.py
@@ -223,8 +223,9 @@ def start_cluster(
                     "id": vol.openstack_id if vol.openstack_id else None,
                     "size": vol.size,
                     "mountPoint": vol.mount_path,
-                    "exists": vol.exists,
-                    "permanent": vol.permanent,
+                    **(
+                        {"exists": True} if vol.exists else {"permanent": vol.permanent}
+                    ),
                 }
                 for vol in worker_volumes
             ]
@@ -237,7 +238,7 @@ def start_cluster(
                 "cloud": "openstack",
                 "sshTimeout": 60,
                 "useMasterAsCompute": False,
-                "useMasterWithPublicIP": self._BIBIGRID_USE_MASTER_WITH_PUBLIC_IP,
+                "useMasterWithPublicIp": self._BIBIGRID_USE_MASTER_WITH_PUBLIC_IP,
                 "dontUploadCredentials": True,
                 "noAllPartition": True,
                 "gateway": {
@@ -256,8 +257,11 @@ def start_cluster(
                                     ),
                                     "size": vol.size,
                                     "mountPoint": vol.mount_path,
-                                    "exists": vol.exists,
-                                    "permanent": vol.permanent,
+                                    **(
+                                        {"exists": True}
+                                        if vol.exists
+                                        else {"permanent": vol.permanent}
+                                    ),
                                 }
                                 for vol in master_volumes
                             ]
diff --git a/simple_vm_client/bibigrid_connector/test_bibigrid_connector.py b/simple_vm_client/bibigrid_connector/test_bibigrid_connector.py
@@ -37,7 +37,6 @@
 METADATA = ClusterInstanceMetadata(user_id="123", project_id="345", project_name="TEST")
 SHARED_VOLUME = ClusterVolume(
     openstack_id="abcd",
-    permanent=True,
     exists=True,
     size=50,
     type="ext4",
@@ -236,7 +235,6 @@ def test_start_cluster(self, mock_logger_info, mock_post):
                         "size": SHARED_VOLUME.size,
                         "mountPoint": SHARED_VOLUME.mount_path,
                         "exists": SHARED_VOLUME.exists,
-                        "permanent": SHARED_VOLUME.permanent,
                     }
                 ]
             }
@@ -248,7 +246,7 @@ def test_start_cluster(self, mock_logger_info, mock_post):
                 "cloud": "openstack",
                 "sshTimeout": 60,
                 "useMasterAsCompute": False,
-                "useMasterWithPublicIP": False,
+                "useMasterWithPublicIp": False,
                 "dontUploadCredentials": True,
                 "noAllPartition": True,
                 "gateway": {"ip": INTERNAL_GATEWAY_IP, "portFunction": PORT_FUNCTION},
diff --git a/simple_vm_client/forc_connector/test_forc_connector.py b/simple_vm_client/forc_connector/test_forc_connector.py
@@ -25,6 +25,8 @@
 REDIS_PORT = 6379
 FORC_API_KEY = "unit_test-key"
 CONFIG_DATA = f"""
+                server:
+                  threads: 32
                 redis:
                   host: {REDIS_HOST}
                   port: {REDIS_PORT}
diff --git a/simple_vm_client/openstack_connector/openstack_connector.py b/simple_vm_client/openstack_connector/openstack_connector.py
@@ -87,6 +87,7 @@ def __init__(self, config_file: str):
         self.APPLICATION_CREDENTIAL_SECRET = ""
         self.USE_APPLICATION_CREDENTIALS: bool = False
         self.NOVA_MICROVERSION = "2.1"
+        self.THREADS = 32
 
         self.load_env_config()
         logger.info(f"Loading config file -- {config_file}")
@@ -109,7 +110,8 @@ def __init__(self, config_file: str):
 
                 # Configure session settings as needed
                 sess.session.connections_pool = True
-                sess.session.connection_pool_size = 32
+
+                sess.session.connection_pool_size = math.ceil(self.THREADS * 1.20)
 
                 # Create the Connection object with the session
                 self.openstack_connection = connection.Connection(session=sess)
@@ -126,7 +128,8 @@ def __init__(self, config_file: str):
                 )
                 sess = session.Session(auth=auth)
                 sess.session.connections_pool = True
-                sess.session.connection_pool_size = 32  # Number of connections to pool
+
+                sess.session.connection_pool_size = math.ceil(self.THREADS * 1.20)
                 self.openstack_connection = connection.Connection(
                     session=sess,
                     compute_api_version=self.NOVA_MICROVERSION,
@@ -158,6 +161,7 @@ def load_config_yml(self, config_file: str) -> None:
             self.FORC_SECURITY_GROUP_ID = cfg["openstack"].get(
                 "forc_security_group_id", None
             )
+            self.THREADS = cfg["server"].get("threads", 32)
             if not self.FORC_SECURITY_GROUP_ID:
                 logger.info("No Forc Security Group defined")
             self.DEFAULT_SECURITY_GROUP_NAME = "defaultSimpleVM"
diff --git a/simple_vm_client/openstack_connector/test_openstack_connector.py b/simple_vm_client/openstack_connector/test_openstack_connector.py
