Pull request #166: Development

Merge in ITB/gitb from development to master

* commit '5e94ce0d4dba0c79dd92b280c2b9f863e3312a6b': (179 commits)
  Update for release 1.24.0
  Minor UI fix
  Replace the current API master key when set
  Re-labelled MASTER API key
  Minor UI improvement
  [ITB-1698] Dash character (-) not counted towards non-character password symbols
  [ITB-1697] Allow administrators of communities not linked to a specific domain to manage any domain
  Corrected titles for XML report previews
  Minor UI fix
  Label correction
  Corrected the ordering of specifications for shared test suites
  Bug fix
  Documentation improvement
  Load user information if needed in route guards
  Updated to gitb-types 1.24.0
  Upgraded GitHub actions
  Baseline script correction
  Corrected deprecation warnings
  DB baseline for release 1.24.0
  Updated ODC to v10.0.4
  ...

.github/workflows/create-release.yml

@@ -13,7 +13,7 @@ jobs:
       contents: write
     steps:
       - uses: actions/checkout@v4
-      - uses: "ncipollo/release-action@v1.13.0"
+      - uses: "ncipollo/release-action@v1.14.0"
         with:
           allowUpdates: true
           omitBodyDuringUpdate: true

README.md

@@ -71,20 +71,20 @@ Follow these steps to build and run the application components for development p
 ## Prerequisites
 
 To build and run the Test Bed's components you need to have the following tools:
-- JDK 17+, used as the base platform for both ``gitb-srv`` and ``gitb-ui``.
+- JDK 21+, used as the base platform for both ``gitb-srv`` and ``gitb-ui``.
 - Maven 3.9+, used to build ``gitb-srv``.
-- SBT 1.9+, used to build ``gitb-ui``.
+- SBT 1.10+, used to build ``gitb-ui``.
 - Scala 2.13+, used to build the backend app of ``gitb-ui``.
-- Node version 20+, used to build the frontend app of ``gitb-ui``.
+- Node version 22+, used to build the frontend app of ``gitb-ui``.
 
 Although not mandatory, the proposed IDE to use is IntelliJ, and VS Code for ``gitb-ui``'s Angular app.
 
 ### Additional Test Bed components
 
 The focus of this README file is the ``gitb-srv`` and ``gitb-ui`` components. To run a complete Test Bed instance
 you will also require at least:
-- A MySQL database (version 8.*) for its persistence.
-- A REDIS instance for the caching of user sessions.
+- A MySQL database (version 8+) for its persistence.
+- A REDIS instance (version 7+) for the caching of user sessions.
 
 Both these instances are set up separately (e.g. via Docker) environment. These can be set up from Docker
 as follows:

docker-compose.yml

@@ -6,7 +6,7 @@ volumes:
 
 services:
    gitb-redis:
-      image: redis:7.0.11
+      image: redis:7.2.5
       restart: unless-stopped
    gitb-mysql:
       build:
@@ -24,8 +24,7 @@ services:
         dockerfile: etc/docker/build/Dockerfile-srv
       restart: unless-stopped
       environment:
-       - gitb.messaging.server-ip-address=localhost
-       - gitb.messaging.callbackURL=http://localhost:8080/itbsrv/MessagingClient
+       - CALLBACK_ROOT_URL=http://localhost:8080/itbsrv
       ports:
        - "8080:8080"
    gitb-ui:

etc/docker/build/Dockerfile-srv

@@ -1,12 +1,12 @@
 # Stage 1: Build app
-FROM maven:3.9.6-amazoncorretto-17 AS builder
+FROM maven:3.9.8-amazoncorretto-21 AS builder
 
 WORKDIR /app
 COPY . /app
 RUN mvn clean install -DskipTests=true -Denv=docker
 
 # Stage 2: Run application
-FROM eclipse-temurin:17-jre-jammy
+FROM eclipse-temurin:21-jre-jammy
 
 RUN mkdir /itbsrv
 COPY --from=builder /app/gitb-testbed-service/target/itbsrv.war /itbsrv

etc/docker/build/Dockerfile-ui

@@ -1,5 +1,5 @@
 # Stage 1: Build app
-FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.9_9_1.9.8_2.13.12 AS builder
+FROM sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.10.0_2.13.14 AS builder
 
 # Install nodejs and npm (to build frontend app)
 ENV NVM_DIR /usr/local/nvm
@@ -42,7 +42,7 @@ RUN mkdir -p /app/tmp \
     && mv /app/tmp/gitb-ui/gitb-1.0-SNAPSHOT /app/tmp/gitb-ui/gitb-ui
 
 # Stage 2: Run application
-FROM eclipse-temurin:17-jre-jammy
+FROM eclipse-temurin:21-jre-jammy
 
 ENV PATH /usr/local/gitb-ui/bin:$PATH
 ENV TESTBED_SERVICE_URL http://gitb-srv:8080/itbsrv/TestbedService

etc/docker/gitb-mysql/Dockerfile

@@ -1,9 +1,14 @@
-FROM mysql:8.4.0
+FROM mysql:8.4.2
 
-ENV MYSQL_ROOT_PASSWORD root
-ENV MYSQL_DATABASE gitb
-ENV MYSQL_USER gitb
-ENV MYSQL_PASSWORD gitb
+ENV MYSQL_ROOT_PASSWORD=root
+ENV MYSQL_DATABASE=gitb
+ENV MYSQL_USER=gitb
+ENV MYSQL_PASSWORD=gitb
 
+COPY gitb-entrypoint.sh /usr/local/bin
 COPY gitb.cnf /etc/mysql/conf.d
-RUN chmod 644 /etc/mysql/conf.d/gitb.cnf
+RUN chmod +x /usr/local/bin/gitb-entrypoint.sh
+RUN chmod 644 /etc/mysql/conf.d/gitb.cnf
+
+ENTRYPOINT ["/usr/local/bin/gitb-entrypoint.sh"]
+CMD ["mysqld"]

etc/docker/gitb-mysql/gitb-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [[ -n "$MYSQL_ROOT_PASSWORD_FILE" ]] ; then
+  unset MYSQL_ROOT_PASSWORD
+fi
+if [[ -n "$MYSQL_PASSWORD_FILE" ]] ; then
+  unset MYSQL_PASSWORD
+fi
+exec /usr/local/bin/docker-entrypoint.sh "$@"

etc/docker/gitb-mysql/gitb.cnf

@@ -1,4 +1,5 @@
 [mysqld]
 
 lower_case_table_names=1
-max_allowed_packet=10485760
+max_allowed_packet=10485760
+mysql_native_password=ON

etc/docker/gitb-srv/Dockerfile

@@ -1,4 +1,4 @@
-FROM eclipse-temurin:17-jre-jammy
+FROM eclipse-temurin:21-jre-jammy
 
 RUN mkdir /itbsrv
 COPY itbsrv.war /itbsrv

etc/docker/gitb-ui/Dockerfile

@@ -1,15 +1,19 @@
-FROM eclipse-temurin:17-jre-jammy
+FROM eclipse-temurin:21-jre-jammy
 
-ENV PATH /usr/local/gitb-ui/bin:$PATH
-ENV TESTBED_SERVICE_URL http://gitb-srv:8080/itbsrv/TestbedService
-ENV TESTBED_CLIENT_URL http://gitb-ui:9090/TestbedClient
-ENV DB_DEFAULT_URL jdbc:mysql://gitb-mysql/gitb?characterEncoding=UTF-8&useUnicode=true&autoReconnect=true&useSSL=false&verifyServerCertificate=false&allowPublicKeyRetrieval=true
-ENV DB_DEFAULT_ROOTURL jdbc:mysql://gitb-mysql/
-ENV REDIS_HOST gitb-redis
+ENV PATH=/usr/local/gitb-ui/bin:$PATH
+ENV TESTBED_SERVICE_URL=http://gitb-srv:8080/itbsrv/TestbedService
+ENV TESTBED_CLIENT_URL_INTERNAL=http://0.0.0.0:9090/TestbedClient
+ENV TESTBED_CLIENT_URL=http://gitb-ui:9090/TestbedClient
+ENV DB_DEFAULT_URL=jdbc:mysql://gitb-mysql/gitb?characterEncoding=UTF-8&useUnicode=true&autoReconnect=true&useSSL=false&verifyServerCertificate=false&allowPublicKeyRetrieval=true
+ENV DB_DEFAULT_ROOTURL=jdbc:mysql://gitb-mysql/
+ENV REDIS_HOST=gitb-redis
 
 COPY gitb-ui /usr/local/gitb-ui
+COPY gitb-entrypoint.sh /usr/local/gitb-ui/bin
+RUN chmod +x /usr/local/gitb-ui/bin/gitb-entrypoint.sh
 
 EXPOSE 9000
 EXPOSE 9090
 WORKDIR /usr/local/gitb-ui
-CMD ["gitb"]
+
+ENTRYPOINT ["/usr/local/gitb-ui/bin/gitb-entrypoint.sh"]

etc/docker/gitb-ui/gitb-entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+cat > /usr/local/gitb-ui/conf/overrides.conf << EOF
+include "application"
+EOF
+
+if [[ -n "$DB_DEFAULT_PASSWORD_FILE" ]] ; then 
+    echo "slick.dbs.default.db.password=$(cat $DB_DEFAULT_PASSWORD_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+    echo "db.default.password=$(cat $DB_DEFAULT_PASSWORD_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$APPLICATION_SECRET_FILE" ]] ; then 
+    echo "play.http.secret.key=$(cat $APPLICATION_SECRET_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$MASTER_PASSWORD_FILE" ]] ; then 
+    echo "masterPassword=$(cat $MASTER_PASSWORD_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$AUTOMATION_API_MASTER_KEY_FILE" ]] ; then
+    echo "masterApiKey=$(cat $AUTOMATION_API_MASTER_KEY_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$HMAC_KEY_FILE" ]] ; then
+    echo "hmac.key=$(cat $HMAC_KEY_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$DATA_ARCHIVE_KEY_FILE" ]] ; then 
+    echo "dataArchive.key=$(cat $DATA_ARCHIVE_KEY_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$PROXY_SERVER_AUTH_PASSWORD_FILE" ]] ; then 
+    echo "proxy.auth.password=$(cat $PROXY_SERVER_AUTH_PASSWORD_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+if [[ -n "$EMAIL_SMTP_AUTH_PASSWORD_FILE" ]] ; then 
+    echo "email.smtp.auth.password=$(cat $EMAIL_SMTP_AUTH_PASSWORD_FILE)" >> /usr/local/gitb-ui/conf/overrides.conf;
+fi
+
+exec gitb -Dconfig.file=/usr/local/gitb-ui/conf/overrides.conf

etc/owasp-suppressions.xml

@@ -35,4 +35,32 @@
         <packageUrl regex="true">^pkg:maven/org\.springframework/spring\-web@.*$</packageUrl>
         <vulnerabilityName>CVE-2024-22262</vulnerabilityName>
     </suppress>
+    <suppress>
+        <notes><![CDATA[
+   file name: jaxb-core-4.0.5.jar
+   ]]></notes>
+        <packageUrl regex="true">^pkg:maven/com\.sun\.xml\.bind/jaxb-core@.*$</packageUrl>
+        <cve>CVE-2024-9329</cve>
+    </suppress>
+    <suppress>
+        <notes><![CDATA[
+   file name: jaxb-core-4.0.5.jar
+   ]]></notes>
+        <packageUrl regex="true">^pkg:maven/org\.glassfish\.jaxb/jaxb-core@.*$</packageUrl>
+        <cve>CVE-2024-9329</cve>
+    </suppress>
+    <suppress>
+        <notes><![CDATA[
+   file name: jaxb-impl-4.0.4.jar (shaded: org.glassfish.jaxb:jaxb-runtime:4.0.4)
+   ]]></notes>
+        <packageUrl regex="true">^pkg:maven/org\.glassfish\.jaxb/jaxb-runtime@.*$</packageUrl>
+        <cve>CVE-2024-9329</cve>
+    </suppress>
+    <suppress>
+        <notes><![CDATA[
+   file name: txw2-4.0.5.jar
+   ]]></notes>
+        <packageUrl regex="true">^pkg:maven/org\.glassfish\.jaxb/txw2@.*$</packageUrl>
+        <cve>CVE-2024-9329</cve>
+    </suppress>
 </suppressions>

gitb-core/src/main/java/com/gitb/types/DataTypeFactory.java

@@ -260,7 +260,7 @@ public DataType create(Variable variable) {
         try {
             //Primitive Types
             if (data instanceof PrimitiveType) {
-                if (variable.getValue().size() > 0) {
+                if (!variable.getValue().isEmpty()) {
                     data.deserialize(variable.getValue().get(0).getValue().getBytes());
                 }
             }
@@ -287,7 +287,7 @@ else if (data instanceof ContainerType) {
             }
             //Complex Types
             else {
-                if(variable.getValue().size() > 0) {
+                if(!variable.getValue().isEmpty()) {
                     data.deserialize(variable.getValue().get(0).getValue().getBytes());
                 }
             }

gitb-core/src/main/java/com/gitb/utils/XMLUtils.java

@@ -97,12 +97,9 @@ Use a factory that disables XML External Entity (XXE) attacks.
          This cannot be done by defining a bean since the XMLInputFactory
          is not thread safe.
          */
-        XMLInputFactory xif = XMLInputFactory.newFactory();
-        xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
-        xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
         XMLStreamReader xsr;
         try {
-            xsr = xif.createXMLStreamReader(source);
+            xsr = getSecureXMLInputFactory().createXMLStreamReader(source);
         } catch (XMLStreamException e) {
             throw new IllegalStateException(e);
         }
@@ -412,7 +409,7 @@ public static TransformerFactory getSecureTransformerFactory() {
     public static XMLInputFactory getSecureXMLInputFactory() {
         XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
         xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
-        xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", false);
+        xmlInputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
         return xmlInputFactory;
     }
 

gitb-engine/src/main/java/com/gitb/engine/TestEngineConfiguration.java

@@ -9,6 +9,9 @@
 import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.apache.commons.lang3.StringUtils;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Objects;
 
 /**
@@ -119,7 +122,7 @@ public static void load() {
 			TEST_ID_PARAMETER = System.getenv().getOrDefault("remote.testcase.test-id.parameter", config.getString("remote.testcase.test-id.parameter"));
 			RESOURCE_ID_PARAMETER = System.getenv().getOrDefault("remote.testcase.resource-id.parameter", config.getString("remote.testcase.resource-id.parameter"));
 			// Configure also the HMAC information used to authorize remote calls.
-			String hmacKey = System.getenv().getOrDefault("HMAC_KEY", "devKey");
+			String hmacKey = getFromFileConfigOrEnvironment("HMAC_KEY", "devKey");
 			String hmacKeyWindow = System.getenv().getOrDefault("HMAC_WINDOW", "10000");
 			HmacUtils.configure(hmacKey, Long.valueOf(hmacKeyWindow));
 			// Remote test case repository - end.
@@ -131,6 +134,21 @@ public static void load() {
 		}
 	}
 
+	private static String getFromFileConfigOrEnvironment(String baseName, String defaultValue) {
+		String filePathName = baseName+"_FILE";
+		if (System.getenv().containsKey(filePathName)) {
+			// Load from file.
+            try {
+                return Files.readString(Path.of(System.getenv(filePathName)));
+            } catch (IOException e) {
+                throw new IllegalStateException("Error reading file", e);
+            }
+        } else {
+			// Load from environment variable or the default.
+			return System.getenv().getOrDefault(baseName, defaultValue);
+		}
+	}
+
 	private static String inferCallbackURL(String endpointName, String referenceCallbackURL) {
 		int index = referenceCallbackURL.lastIndexOf('/');
 		if (index >= 0) {

gitb-engine/src/main/java/com/gitb/engine/actors/ActorSystem.java

@@ -12,6 +12,7 @@ public class ActorSystem {
 	public static final String ACTOR_SYSTEM_NAME = "test-engine-as";
 
 	public static final String BLOCKING_DISPATCHER = "blocking-processor-dispatcher";
+	public static final String BLOCKING_IO_DISPATCHER = "blocking-io-dispatcher";
 
 	private org.apache.pekko.actor.ActorSystem system;
 

gitb-engine/src/main/java/com/gitb/engine/actors/processors/AbstractProcessorActor.java

@@ -11,6 +11,7 @@
 import org.apache.pekko.dispatch.Futures;
 import org.apache.pekko.dispatch.OnFailure;
 import org.apache.pekko.dispatch.OnSuccess;
+import scala.concurrent.ExecutionContext;
 import scala.concurrent.Future;
 import scala.concurrent.Promise;
 
@@ -70,7 +71,7 @@ protected void start() {
 				processing();
 
 				return processor.process(step);
-			}, getContext().dispatcher());
+			}, stepDispatcher());
 
 			future.foreach(new OnSuccess<>() {
 
@@ -89,6 +90,10 @@ public void onFailure(Throwable failure) {
 		}
 	}
 
+	protected ExecutionContext stepDispatcher() {
+		return getContext().getDispatcher();
+	}
+
 	@Override
 	protected void stop() {
         if(promise != null) {

gitb-engine/src/main/java/com/gitb/engine/actors/processors/AbstractTestStepActor.java

@@ -167,7 +167,6 @@ public void onReceive(Object message) {
 				throw new GITBEngineInternalError("Invalid command [" + message.getClass().getName() + "]");
 			}
 		} catch (Exception e) {
-			logger.error(addMarker(), "Processing caught an exception", e);
 			error(e);
 		}
 	}