Merge pull request #493 from mikepenz/feature/479

JUnit retries are handled as passes
mikepenz · Jan 28, 2022 · 6bf2676 · 6bf2676
2 parents 738031d + 0aa07f5
commit 6bf2676
Show file tree

Hide file tree

Showing 7 changed files with 156 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -81,6 +81,7 @@ jobs:
 | `commit`          | Optional. The commit SHA to update the status. This is useful when you run it with `workflow_run`.                                                                 |
 | `fail_on_failure` | Optional. Fail the build in case of a test failure.                                                                                                                |
 | `require_tests`   | Optional. Fail if no test are found..                                                                                                                              |
+| `check_retries`         | Optional. If a testcase is retried, ignore the original failure.                                                                                             |
 | `check_title_template`  | Optional. Template to configure the title format. Placeholders: ${{FILE_NAME}}, ${{SUITE_NAME}}, ${{TEST_NAME}}.                                             |
 | `summary`         | Optional. Additional text to summary output                                                                                                                        |
 

diff --git a/__tests__/testParser.test.ts b/__tests__/testParser.test.ts
@@ -426,7 +426,7 @@ action.surefire.report.email.InvalidEmailAddressException: Invalid email address
     });
 
     it('parse mocha test case, custom title template', async () => {
-        const { count, skipped, annotations } = await parseFile('test_results/mocha/mocha.xml', '*', true, ['/build/', '/__pycache__/'], '${{TEST_NAME}}');
+        const { count, skipped, annotations } = await parseFile('test_results/mocha/mocha.xml', '*', true, false, ['/build/', '/__pycache__/'], '${{TEST_NAME}}');
 
         expect(count).toBe(1);
         expect(skipped).toBe(0);
@@ -482,4 +482,55 @@ action.surefire.report.email.InvalidEmailAddressException: Invalid email address
             }
         ]);
     });
+
+    it('should handle retries', async () => {
+        const { count, skipped, annotations } = await parseFile('test_results/junit-web-test/expectedRetries.xml', '', false, true, ['/build/', '/__pycache__/']);
+
+        expect(count).toBe(7);
+        expect(skipped).toBe(1);
+        expect(annotations).toStrictEqual([
+            {
+                path: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js",
+                start_line: 15,
+                end_line: 15,
+                start_column: 0,
+                end_column: 0,
+                annotation_level: "failure",
+                title: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js.asserts error",
+                message: "expected false to be true",
+                raw_details: "AssertionError: expected false to be true\n  at o.<anonymous> (packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js:15:29)",
+            }
+        ]);
+    });
+
+    it('there should be two errors if retries are not handled', async () => {
+        const { count, skipped, annotations } = await parseFile('test_results/junit-web-test/expectedRetries.xml', '', false);
+
+        expect(count).toBe(8);
+        expect(skipped).toBe(1);
+        expect(annotations).toStrictEqual([
+            {
+                path: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js",
+                start_line: 15,
+                end_line: 15,
+                start_column: 0,
+                end_column: 0,
+                annotation_level: "failure",
+                title: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js.asserts error",
+                message: "expected false to be true",
+                raw_details: "AssertionError: expected false to be true\n  at o.<anonymous> (packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js:15:29)",
+            },
+            {
+                annotation_level: "failure",
+                end_column: 0,
+                end_line: 15,
+                message: "this is flaky, so is retried",
+                path: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js",
+                raw_details: "AssertionError: expected false to be true\n  at o.<anonymous> (packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js:15:29)",
+                start_column: 0,
+                start_line: 15,
+                title: "packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js.retried flaky test",
+            }
+        ]);
+    });
 });
diff --git a/dist/index.js b/dist/index.js
diff --git a/dist/index.js.map b/dist/index.js.map
diff --git a/src/main.ts b/src/main.ts
@@ -29,6 +29,7 @@ export async function run(): Promise<void> {
     const excludeSources = core.getInput('exclude_sources')
       ? core.getInput('exclude_sources').split(',')
       : []
+    const checkRetries = core.getInput('check_retries') === 'true'
 
     core.endGroup()
     core.startGroup(`📦 Process test results`)
@@ -37,6 +38,7 @@ export async function run(): Promise<void> {
       reportPaths,
       suiteRegex,
       includePassed,
+      checkRetries,
       excludeSources,
       checkTitleTemplate
     )

diff --git a/src/testParser.ts b/src/testParser.ts
@@ -131,6 +131,7 @@ export async function parseFile(
   file: string,
   suiteRegex = '',
   includePassed = false,
+  checkRetries = false,
   excludeSources: string[] = ['/build/', '/__pycache__/'],
   checkTitleTemplate: string | undefined = undefined
 ): Promise<TestResult> {
@@ -144,6 +145,7 @@ export async function parseFile(
     '',
     suiteRegex,
     includePassed,
+    checkRetries,
     excludeSources,
     checkTitleTemplate
   )
@@ -155,6 +157,7 @@ async function parseSuite(
   parentName: string,
   suiteRegex: string,
   includePassed = false,
+  checkRetries = false,
   excludeSources: string[],
   checkTitleTemplate: string | undefined = undefined
 ): Promise<TestResult> {
@@ -196,6 +199,7 @@ async function parseSuite(
       suiteName,
       suiteRegex,
       includePassed,
+      checkRetries,
       excludeSources,
       checkTitleTemplate
     )
@@ -207,11 +211,37 @@ async function parseSuite(
       continue
     }
 
-    const testcases = Array.isArray(testsuite.testcase)
+    let testcases = Array.isArray(testsuite.testcase)
       ? testsuite.testcase
       : testsuite.testcase
       ? [testsuite.testcase]
       : []
+
+    if (checkRetries) {
+      // identify duplicates, in case of flaky tests, and remove them
+      const testcaseMap = new Map<string, any>()
+      for (const testcase of testcases) {
+        const key = testcase._attributes.name
+        if (testcaseMap.get(key) !== undefined) {
+          // testcase with matching name exists
+          const failed = testcase.failure || testcase.error
+          const previousFailed =
+            testcaseMap.get(key).failure || testcaseMap.get(key).error
+          if (failed && !previousFailed) {
+            // previous is a success, drop failure
+            core.debug(`Drop flaky test failure for (1): ${key}`)
+          } else if (!failed && previousFailed) {
+            // previous failed, new one not, replace
+            testcaseMap.set(key, testcase)
+            core.debug(`Drop flaky test failure for (2): ${key}`)
+          }
+        } else {
+          testcaseMap.set(key, testcase)
+        }
+      }
+      testcases = Array.from(testcaseMap.values())
+    }
+
     for (const testcase of testcases) {
       count++
 
@@ -311,6 +341,7 @@ export async function parseTestReports(
   reportPaths: string,
   suiteRegex: string,
   includePassed = false,
+  checkRetries = false,
   excludeSources: string[],
   checkTitleTemplate: string | undefined = undefined
 ): Promise<TestResult> {
@@ -327,6 +358,7 @@ export async function parseTestReports(
       file,
       suiteRegex,
       includePassed,
+      checkRetries,
       excludeSources,
       checkTitleTemplate
     )

diff --git a/test_results/junit-web-test/expectedRetries.xml b/test_results/junit-web-test/expectedRetries.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="Chrome_puppeteer_/packages/test-runner-junit-reporter/test/fixtures/multiple/nested-test.js" id="0" tests="1" skipped="0" errors="0" failures="0" time="0.001">
+    <properties>
+      <property name="test.fileName" value="packages/test-runner-junit-reporter/test/fixtures/multiple/nested-test.js"/>
+      <property name="browser.name" value="Chrome"/>
+      <property name="browser.launcher" value="puppeteer"/>
+    </properties>
+    <testcase name="is equivalent to map (f.g) f a" time="<<computed>>" classname="given a functor f and a function a -&gt; b map f map g f a" file="packages/test-runner-junit-reporter/test/fixtures/multiple/nested-test.js"/>
+  </testsuite>
+  <testsuite name="Chrome_puppeteer_/packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js" id="0" tests="5" skipped="1" errors="1" failures="1" time="0.001">
+    <properties>
+      <property name="test.fileName" value="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js"/>
+      <property name="browser.name" value="Chrome"/>
+      <property name="browser.launcher" value="puppeteer"/>
+    </properties>
+    <testcase name="under addition" time="0.001" classname="real numbers forming a monoid" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js"/>
+    <testcase name="null hypothesis" time="0.001" classname="off-by-one boolean logic errors" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js"/>
+    <testcase name="asserts error" time="0.001" classname="off-by-one boolean logic errors" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js" line="15">
+      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
+  at o.<anonymous> (packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js:15:29)]]></failure>
+    </testcase>
+    <testcase name="retried flaky test" time="0.001" classname="off-by-one boolean logic errors" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js" line="15">
+      <failure message="this is flaky, so is retried" type="AssertionError"><![CDATA[AssertionError: expected false to be true
+  at o.<anonymous> (packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js:15:29)]]></failure>
+    </testcase>
+    <testcase name="tbd: confirm true positive" time="0" classname="off-by-one boolean logic errors" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js">
+      <skipped/>
+    </testcase>
+    <testcase name="retried flaky test" time="0.001" classname="off-by-one boolean logic errors" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js" line="15"/>
+    <testcase name="reports logs to JUnit" time="0.001" classname="logging during a test" file="packages/test-runner-junit-reporter/test/fixtures/multiple/simple-test.js"/>
+  </testsuite>
+</testsuites>