From 2d71b75b2bf0fa2661ac229f34cb4c5df58ff2d6 Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 15:52:17 +0100 Subject: [PATCH 1/6] when e2e test docker-compose fails to start, capture logs from the containers and print --- .../debuggable-nu-designer.override.yml | 1 - e2e-tests/src/test/resources/logback-test.xml | 17 +++++++ ...asedNuInstallationExampleEnvironment.scala | 44 ++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 e2e-tests/src/test/resources/logback-test.xml diff --git a/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml b/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml index c478a56d6c0..205a1e6c611 100644 --- a/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml +++ b/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml @@ -5,5 +5,4 @@ services: - "5005:5005" environment: JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 - JDK_JAVA_OPTIONS: "-Xmx2048M" NUSSKNACKER_LOG_LEVEL: debug diff --git a/e2e-tests/src/test/resources/logback-test.xml b/e2e-tests/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..40ee1ab0765 --- /dev/null +++ b/e2e-tests/src/test/resources/logback-test.xml @@ -0,0 +1,17 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n + + + + + + + + + + + + + diff --git a/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala b/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala index e2dd40a5272..f17ebf949d9 100644 --- a/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala +++ b/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala @@ -4,10 +4,15 @@ import cats.effect.IO import cats.effect.kernel.Resource import cats.effect.unsafe.implicits.global import com.dimafeng.testcontainers.{DockerComposeContainer, ServiceLogConsumer, WaitingForService} +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.model.Container import com.typesafe.scalalogging.LazyLogging import org.slf4j.Logger +import org.slf4j.MarkerFactory.getIMarkerFactory +import org.testcontainers.DockerClientFactory import org.testcontainers.containers.output.Slf4jLogConsumer import org.testcontainers.containers.wait.strategy.DockerHealthcheckWaitStrategy +import org.testcontainers.utility.LogUtils import pl.touk.nussknacker.test.MiscUtils._ import pl.touk.nussknacker.test.WithTestHttpClientCreator import pl.touk.nussknacker.test.containers.ContainerExt.toContainerExt @@ -18,7 +23,9 @@ import ujson.Value import java.io.{File => JFile} import java.time.Duration -import scala.util.Try +import java.util +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} class DockerBasedInstallationExampleNuEnvironment( nussknackerImageVersion: String, @@ -46,7 +53,15 @@ class DockerBasedInstallationExampleNuEnvironment( tailChildContainers = false ) { - start() + Try(start()) match { + case Failure(ex) => + // There is no way currently to automatically capture logs from containers before all services from the docker + // compose started. When one of the services is not healthy, there won't be any logs captured. That's why we do + // the capture manually. + captureAllContainerLogs() + throw ex + case Success(()) => + } private val (dockerBasedInstallationExampleClient, closeHandler) = DockerBasedInstallationExampleClient.create(this).allocated.unsafeRunSync() @@ -58,6 +73,31 @@ class DockerBasedInstallationExampleNuEnvironment( super.stop() } + private def captureAllContainerLogs() = { + val dockerClient = DockerClientFactory.lazyClient() + getNuDockerComposeContainers(dockerClient).foreach { container => + val logs = LogUtils.getOutput(dockerClient, container.getId) + slf4jLogger.info(getIMarkerFactory.getMarker(container.getNames.mkString(",")), logs) + } + } + + private def getNuDockerComposeContainers(dockerClient: DockerClient) = { + dockerClient + .listContainersCmd() + .exec() + .asInstanceOf[util.ArrayList[Container]] + .asScala + .filter { container => + // dummy method of how to distinguish if the container is Nu docker-compose related container + container.labels.asScala.get("com.docker.compose.project.working_dir") match { + case Some(value) => + value.contains("nussknacker") + case None => false + } + } + .toList + } + } object DockerBasedInstallationExampleNuEnvironment extends LazyLogging { From 8fa94d9b9b450fa3e443ed63c2c43bb23387a9d6 Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 16:29:44 +0100 Subject: [PATCH 2/6] fix --- e2e-tests/src/test/resources/logback-test.xml | 17 ----------------- .../src/main/resources/logback-test.xml | 4 ++++ 2 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 e2e-tests/src/test/resources/logback-test.xml diff --git a/e2e-tests/src/test/resources/logback-test.xml b/e2e-tests/src/test/resources/logback-test.xml deleted file mode 100644 index 40ee1ab0765..00000000000 --- a/e2e-tests/src/test/resources/logback-test.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - - - - - - - diff --git a/utils/test-utils/src/main/resources/logback-test.xml b/utils/test-utils/src/main/resources/logback-test.xml index 6211e9153da..76cc6bc936c 100644 --- a/utils/test-utils/src/main/resources/logback-test.xml +++ b/utils/test-utils/src/main/resources/logback-test.xml @@ -70,6 +70,10 @@ + + + + From f6a564a6387791702e241cb60b957171726780f8 Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 16:42:58 +0100 Subject: [PATCH 3/6] review fixes --- utils/test-utils/src/main/resources/logback-test.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test-utils/src/main/resources/logback-test.xml b/utils/test-utils/src/main/resources/logback-test.xml index 76cc6bc936c..79fb9f1fc41 100644 --- a/utils/test-utils/src/main/resources/logback-test.xml +++ b/utils/test-utils/src/main/resources/logback-test.xml @@ -73,7 +73,7 @@ - + From 8f1780950de1044a6c6c8b7c9d636104ca0e91d0 Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 17:04:08 +0100 Subject: [PATCH 4/6] fix benchmark --- .../DockerBasedNuInstallationExampleEnvironment.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala b/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala index f17ebf949d9..321ffb0791d 100644 --- a/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala +++ b/utils/test-utils/src/main/scala/pl/touk/nussknacker/test/installationexample/DockerBasedNuInstallationExampleEnvironment.scala @@ -127,7 +127,7 @@ class DockerBasedInstallationExampleClient private ( def deployAndWaitForRunningState(scenarioName: String): Unit = { bootstrapSetupService.executeBash( - s"""/app/utils/nu/deploy-scenario-and-wait-for-running-state.sh "$scenarioName" """ + s"""/app/utils/nu/deploy-scenario-and-wait-for-deployed-state.sh "$scenarioName" """ ) } From ce3a78e233c07990332c0d8ff73ab435d6e6130d Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 17:13:46 +0100 Subject: [PATCH 5/6] fix --- .../test/resources/bootstrap-setup-scenarios.override.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml b/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml index 38b70fa7b0e..80d324b23c3 100644 --- a/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml +++ b/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml @@ -1,5 +1,13 @@ services: + designer: + environment: + JDK_JAVA_OPTIONS: "-Xmx2048M" + deploy: + resources: + limits: + memory: 2048M + bootstrap-setup: volumes: - ../../e2e-tests/src/test/resources/detect-large-transactions:/scenario-examples/detect-large-transactions From cc91a89c6fdf8eaef411477388e1ef5eeb4f25a0 Mon Sep 17 00:00:00 2001 From: MK Software Date: Fri, 24 Jan 2025 18:09:01 +0100 Subject: [PATCH 6/6] revert --- .../test/resources/bootstrap-setup-scenarios.override.yml | 8 -------- .../test/resources/debuggable-nu-designer.override.yml | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml b/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml index 80d324b23c3..38b70fa7b0e 100644 --- a/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml +++ b/e2e-tests/src/test/resources/bootstrap-setup-scenarios.override.yml @@ -1,13 +1,5 @@ services: - designer: - environment: - JDK_JAVA_OPTIONS: "-Xmx2048M" - deploy: - resources: - limits: - memory: 2048M - bootstrap-setup: volumes: - ../../e2e-tests/src/test/resources/detect-large-transactions:/scenario-examples/detect-large-transactions diff --git a/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml b/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml index 205a1e6c611..c478a56d6c0 100644 --- a/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml +++ b/e2e-tests/src/test/resources/debuggable-nu-designer.override.yml @@ -5,4 +5,5 @@ services: - "5005:5005" environment: JAVA_TOOL_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + JDK_JAVA_OPTIONS: "-Xmx2048M" NUSSKNACKER_LOG_LEVEL: debug