From f71f69919bdd472ab0709a27e73d10382aff8b1b Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Thu, 10 Oct 2024 14:50:58 -0700 Subject: [PATCH] Add support for @AfterAll in XML report Writing a simple test like ``` package com.library; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; public class AlwaysFailTest { @Test public void doNothingTest() { System.out.println("Hi there!"); } @AfterAll public static void alwaysFail() { throw new RuntimeException("Always failing."); } } ``` Produces an XML that seems to look like it succeeds. ```xml ``` Bazel itself correctly identifies it fails. The Junit4 reporter also included with Bazel natively correclty reports the failure in the XML output. Augment the Junit listener to add support for static methods and add them to the JUnit output. Doing so produces the following XML. ```xml ``` --- MODULE.bazel | 1 + WORKSPACE | 1 + contrib_rules_jvm_tests_install.json | 136 +++++++++++++++++- .../junit5/BazelJUnitOutputListener.java | 6 +- .../contrib_rules_jvm/junit5/TestData.java | 21 +++ .../contrib_rules_jvm/junit5/BUILD.bazel | 1 + ...java => BazelJUnitOutputListenerTest.java} | 74 +++++++++- 7 files changed, 235 insertions(+), 5 deletions(-) mode change 100644 => 100755 contrib_rules_jvm_tests_install.json rename java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/{BazelJUnitOuputListenerTest.java => BazelJUnitOutputListenerTest.java} (82%) diff --git a/MODULE.bazel b/MODULE.bazel index 8bc69be3..7eeb7bfa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -228,6 +228,7 @@ dev_maven.install( "org.junit.platform:junit-platform-suite:1.8.2", "org.junit.platform:junit-platform-suite-api:1.8.2", "org.junit.platform:junit-platform-suite-engine:1.8.2", + "org.junit.platform:junit-platform-testkit:1.8.2", "org.junit.vintage:junit-vintage-engine:5.8.2", "org.mockito:mockito-core:4.8.1", ], diff --git a/WORKSPACE b/WORKSPACE index f6b23149..09fbc8bd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -54,6 +54,7 @@ maven_install( "org.junit.platform:junit-platform-suite:1.8.2", "org.junit.platform:junit-platform-suite-api:1.8.2", "org.junit.platform:junit-platform-suite-engine:1.8.2", + "org.junit.platform:junit-platform-testkit:1.8.2", "org.junit.vintage:junit-vintage-engine:5.8.2", "org.mockito:mockito-core:4.8.1", ], diff --git a/contrib_rules_jvm_tests_install.json b/contrib_rules_jvm_tests_install.json old mode 100644 new mode 100755 index 52ce5399..e56535b0 --- a/contrib_rules_jvm_tests_install.json +++ b/contrib_rules_jvm_tests_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -579211453, - "__RESOLVED_ARTIFACTS_HASH": -1476897549, + "__INPUT_ARTIFACTS_HASH": -110331756, + "__RESOLVED_ARTIFACTS_HASH": 1220653889, "artifacts": { "junit:junit": { "shasums": { @@ -31,6 +31,13 @@ }, "version": "1.1.2" }, + "org.assertj:assertj-core": { + "shasums": { + "jar": "d749db58c2bd353f1c03541d747b753931d4b84da8e48993ef51efe8694b4ed7", + "sources": "d0384d378df4391392bbdf06691aa48b3ac3bc5f37c223e6466a75a03d54fee4" + }, + "version": "3.20.2" + }, "org.hamcrest:hamcrest-core": { "shasums": { "jar": "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9", @@ -115,6 +122,13 @@ }, "version": "1.8.2" }, + "org.junit.platform:junit-platform-testkit": { + "shasums": { + "jar": "94d6fb9b1d4e7bb3d6a967b21f1cf4f7cf84455d602cccf614a186d936929b08", + "sources": "485fac0cc55c14db0c10dff3e42351b10e0bfe5d337892efe8e5b5fcb672f752" + }, + "version": "1.8.2" + }, "org.junit.vintage:junit-vintage-engine": { "shasums": { "jar": "ebd567b84e380d5373c47de3c9616d84f7bef91f9f8a8e7fc925be68240c1ba4", @@ -198,6 +212,12 @@ "org.junit.platform:junit-platform-suite-api", "org.junit.platform:junit-platform-suite-commons" ], + "org.junit.platform:junit-platform-testkit": [ + "org.apiguardian:apiguardian-api", + "org.assertj:assertj-core", + "org.junit.platform:junit-platform-launcher", + "org.opentest4j:opentest4j" + ], "org.junit.vintage:junit-vintage-engine": [ "junit:junit", "org.apiguardian:apiguardian-api", @@ -294,6 +314,69 @@ "org.apiguardian:apiguardian-api": [ "org.apiguardian.api" ], + "org.assertj:assertj-core": [ + "org.assertj.core.annotations", + "org.assertj.core.api", + "org.assertj.core.api.exception", + "org.assertj.core.api.filter", + "org.assertj.core.api.iterable", + "org.assertj.core.api.junit.jupiter", + "org.assertj.core.api.recursive.comparison", + "org.assertj.core.condition", + "org.assertj.core.configuration", + "org.assertj.core.data", + "org.assertj.core.description", + "org.assertj.core.error", + "org.assertj.core.error.array2d", + "org.assertj.core.error.future", + "org.assertj.core.error.uri", + "org.assertj.core.extractor", + "org.assertj.core.groups", + "org.assertj.core.internal", + "org.assertj.core.internal.bytebuddy", + "org.assertj.core.internal.bytebuddy.agent.builder", + "org.assertj.core.internal.bytebuddy.asm", + "org.assertj.core.internal.bytebuddy.build", + "org.assertj.core.internal.bytebuddy.description", + "org.assertj.core.internal.bytebuddy.description.annotation", + "org.assertj.core.internal.bytebuddy.description.enumeration", + "org.assertj.core.internal.bytebuddy.description.field", + "org.assertj.core.internal.bytebuddy.description.method", + "org.assertj.core.internal.bytebuddy.description.modifier", + "org.assertj.core.internal.bytebuddy.description.type", + "org.assertj.core.internal.bytebuddy.dynamic", + "org.assertj.core.internal.bytebuddy.dynamic.loading", + "org.assertj.core.internal.bytebuddy.dynamic.scaffold", + "org.assertj.core.internal.bytebuddy.dynamic.scaffold.inline", + "org.assertj.core.internal.bytebuddy.dynamic.scaffold.subclass", + "org.assertj.core.internal.bytebuddy.implementation", + "org.assertj.core.internal.bytebuddy.implementation.attribute", + "org.assertj.core.internal.bytebuddy.implementation.auxiliary", + "org.assertj.core.internal.bytebuddy.implementation.bind", + "org.assertj.core.internal.bytebuddy.implementation.bind.annotation", + "org.assertj.core.internal.bytebuddy.implementation.bytecode", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.assign", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.assign.primitive", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.assign.reference", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.collection", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.constant", + "org.assertj.core.internal.bytebuddy.implementation.bytecode.member", + "org.assertj.core.internal.bytebuddy.jar.asm", + "org.assertj.core.internal.bytebuddy.jar.asm.commons", + "org.assertj.core.internal.bytebuddy.jar.asm.signature", + "org.assertj.core.internal.bytebuddy.matcher", + "org.assertj.core.internal.bytebuddy.pool", + "org.assertj.core.internal.bytebuddy.utility", + "org.assertj.core.internal.bytebuddy.utility.privilege", + "org.assertj.core.internal.bytebuddy.utility.visitor", + "org.assertj.core.matcher", + "org.assertj.core.presentation", + "org.assertj.core.util", + "org.assertj.core.util.diff", + "org.assertj.core.util.diff.myers", + "org.assertj.core.util.introspection", + "org.assertj.core.util.xml" + ], "org.hamcrest:hamcrest-core": [ "org.hamcrest", "org.hamcrest.core", @@ -380,6 +463,9 @@ "org.junit.platform:junit-platform-suite-engine": [ "org.junit.platform.suite.engine" ], + "org.junit.platform:junit-platform-testkit": [ + "org.junit.platform.testkit.engine" + ], "org.junit.vintage:junit-vintage-engine": [ "org.junit.vintage.engine", "org.junit.vintage.engine.descriptor", @@ -479,6 +565,8 @@ "net.bytebuddy:byte-buddy:jar:sources", "org.apiguardian:apiguardian-api", "org.apiguardian:apiguardian-api:jar:sources", + "org.assertj:assertj-core", + "org.assertj:assertj-core:jar:sources", "org.hamcrest:hamcrest-core", "org.hamcrest:hamcrest-core:jar:sources", "org.junit.jupiter:junit-jupiter-api", @@ -503,6 +591,8 @@ "org.junit.platform:junit-platform-suite-engine", "org.junit.platform:junit-platform-suite-engine:jar:sources", "org.junit.platform:junit-platform-suite:jar:sources", + "org.junit.platform:junit-platform-testkit", + "org.junit.platform:junit-platform-testkit:jar:sources", "org.junit.vintage:junit-vintage-engine", "org.junit.vintage:junit-vintage-engine:jar:sources", "org.mockito:mockito-core", @@ -513,5 +603,47 @@ "org.opentest4j:opentest4j:jar:sources" ] }, + "services": { + "org.junit.jupiter:junit-jupiter-engine": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.jupiter.engine.JupiterTestEngine" + ] + }, + "org.junit.jupiter:junit-jupiter-engine:jar:sources": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.jupiter.engine.JupiterTestEngine" + ] + }, + "org.junit.platform:junit-platform-launcher": { + "org.junit.platform.launcher.TestExecutionListener": [ + "org.junit.platform.launcher.listeners.UniqueIdTrackingListener" + ] + }, + "org.junit.platform:junit-platform-launcher:jar:sources": { + "org.junit.platform.launcher.TestExecutionListener": [ + "org.junit.platform.launcher.listeners.UniqueIdTrackingListener" + ] + }, + "org.junit.platform:junit-platform-suite-engine": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.platform.suite.engine.SuiteTestEngine" + ] + }, + "org.junit.platform:junit-platform-suite-engine:jar:sources": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.platform.suite.engine.SuiteTestEngine" + ] + }, + "org.junit.vintage:junit-vintage-engine": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.vintage.engine.VintageTestEngine" + ] + }, + "org.junit.vintage:junit-vintage-engine:jar:sources": { + "org.junit.platform.engine.TestEngine": [ + "org.junit.vintage.engine.VintageTestEngine" + ] + } + }, "version": "2" } diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListener.java b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListener.java index f132c2eb..9ac97de5 100644 --- a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListener.java +++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListener.java @@ -109,7 +109,9 @@ private Map> matchTestCasesToSuites_locked( // results List segments = testCase.getId().getUniqueIdObject().getSegments(); - if (segments.size() == 3 || segments.size() == 5) { + if (segments.size() == 2) { + parent = results.get(testCase.getId().getUniqueIdObject()); + } else if (segments.size() == 3 || segments.size() == 5) { // get class / test data up one level parent = testCase @@ -161,7 +163,7 @@ private List findTestCases_locked() { // are identified by the fact that they have no child test cases in the // test plan, or they are marked as tests. TestIdentifier id = result.getId(); - return id.isTest() || testPlan.getChildren(id).isEmpty(); + return id.getSource() != null || id.isTest() || testPlan.getChildren(id).isEmpty(); }) .collect(Collectors.toList()); } diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestData.java b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestData.java index 4feb54a6..c16fa0e9 100644 --- a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestData.java +++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/TestData.java @@ -146,4 +146,25 @@ public boolean isDynamic() { public Instant getStarted() { return this.started; } + + @Override + public String toString() { + return "TestData{" + + "id=" + + id + + ", reportEntries=" + + reportEntries + + ", started=" + + started + + ", finished=" + + finished + + ", reason='" + + reason + + '\'' + + ", result=" + + result + + ", dynamic=" + + dynamic + + '}'; + } } diff --git a/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel b/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel index 27b1bd99..99b697b9 100644 --- a/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel +++ b/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel @@ -38,6 +38,7 @@ java_test_suite( artifact("org.junit.jupiter:junit-jupiter-api", "contrib_rules_jvm_tests"), artifact("org.junit.jupiter:junit-jupiter-params", "contrib_rules_jvm_tests"), artifact("org.junit.platform:junit-platform-engine", "contrib_rules_jvm_tests"), + artifact("org.junit.platform:junit-platform-testkit", "contrib_rules_jvm_tests"), artifact("org.mockito:mockito-core", "contrib_rules_jvm_tests"), artifact("org.opentest4j:opentest4j", "contrib_rules_jvm_tests"), ] + junit5_vintage_deps("contrib_rules_jvm_tests"), diff --git a/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOuputListenerTest.java b/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListenerTest.java similarity index 82% rename from java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOuputListenerTest.java rename to java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListenerTest.java index 820c517e..24001d8b 100644 --- a/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOuputListenerTest.java +++ b/java/test/com/github/bazel_contrib/contrib_rules_jvm/junit5/BazelJUnitOutputListenerTest.java @@ -5,29 +5,40 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY; import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; +import java.nio.file.Files; import java.util.Collection; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.core.LauncherConfig; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.testkit.engine.EngineTestKit; import org.mockito.Mockito; import org.opentest4j.TestAbortedException; import org.w3c.dom.Document; @@ -37,11 +48,72 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException; -public class BazelJUnitOuputListenerTest { +public class BazelJUnitOutputListenerTest { private TestDescriptor testDescriptor = new StubbedTestDescriptor(createId("descriptors")); private TestIdentifier identifier = TestIdentifier.from(testDescriptor); + /** This latch is used in TestAfterAllFails for testAfterAllFailuresAreReported */ + private static final AtomicBoolean causeFailure = new AtomicBoolean(false); + + static final class TestAfterAllFails { + + @SuppressFBWarnings(value = "THROWS_METHOD_THROWS_RUNTIMEEXCEPTION") + @AfterAll + static void afterAll() { + if (causeFailure.get()) { + throw new RuntimeException("I always fail."); + } + } + + @Test + void test() {} + } + + @Test + public void testAfterAllFailuresAreReported() throws IOException { + causeFailure.set(true); + + // First let's do a sanity test that we have the expected failures for the @AfterAll + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass(TestAfterAllFails.class)) + .execute() + .containerEvents() + .assertStatistics(stats -> stats.skipped(0).started(2).succeeded(1).aborted(0).failed(1)); + + // Now let's run the same test. Unfortunately we cannot use EngineTestKit since it has no way + // to register a listener. + File xmlFile = File.createTempFile("junit-report", "xml"); + BazelJUnitOutputListener listener = new BazelJUnitOutputListener(xmlFile.toPath()); + LauncherConfig config = LauncherConfig.builder().addTestExecutionListeners(listener).build(); + + Launcher launcher = LauncherFactory.create(config); + + LauncherDiscoveryRequestBuilder request = + LauncherDiscoveryRequestBuilder.request().selectors(selectClass(TestAfterAllFails.class)); + + launcher.discover(request.build()); + + launcher.execute(request.build()); + listener.close(); + + // now write an assertion to validate the XML file has an error + String[] expectedStrings = { + "", "failures=\"1\"", + }; + + // Useful for debugging the expected output + // System.out.println(Files.readString(xmlFile.toPath())); + + for (String expected : expectedStrings) { + assertTrue( + Files.lines(xmlFile.toPath()).anyMatch(line -> line.contains(expected)), + "Expected to find " + expected + " in " + xmlFile); + } + + causeFailure.set(false); + } + @Test public void testResultCanBeDisabled() { // Note: we do not call `markFinished` so the TestResult is null. This is what happens when