optimize: Introduce automated flaky test tracking like OpenSearch

.github/scripts/parse_failed_tests.py

@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import xml.etree.ElementTree as ET
+import json
+
+def find_testcases_with_status(report_dir):
+    """
+    all xml files under report_dir, parse results
+    return dict {(classname, testname): 'passed'/'failed'}
+    """
+    results = {}
+
+    for root, _, files in os.walk(report_dir):
+        for file in files:
+            if not file.endswith(".xml"):
+                continue
+            path = os.path.join(root, file)
+            try:
+                tree = ET.parse(path)
+                root_elem = tree.getroot()
+                for testcase in root_elem.iter("testcase"):
+                    classname = testcase.attrib.get("classname")
+                    testname = testcase.attrib.get("name")
+
+                    failed = testcase.find("failure") is not None or testcase.find("error") is not None
+                    status = "failed" if failed else "passed"
+                    results[(classname, testname)] = status
+            except Exception as e:
+                print(f"Warning: Failed to parse {path}: {e}", file=sys.stderr)
+    return results
+
+
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: parse_failed_tests.py <report_dir_1> <report_dir_2>")
+        sys.exit(1)
+
+    report_dir_1 = sys.argv[1]
+    report_dir_2 = sys.argv[2]
+
+    results_1 = find_testcases_with_status(report_dir_1)
+    results_2 = find_testcases_with_status(report_dir_2)
+
+    flaky_tests = []
+    for test_id, status_1 in results_1.items():
+        status_2 = results_2.get(test_id, "passed")
+        if status_1 == "failed" and status_2 == "passed":
+            flaky_tests.append(test_id)
+
+    if flaky_tests:
+        print(json.dumps([f"{c}.{t}" for c,t in flaky_tests]))
+    else:
+        print("[]")
+
+if __name__ == "__main__":
+    main()

.github/workflows/build.yml

@@ -66,6 +66,8 @@ jobs:
       # step 4
       - name: "Print maven version"
         run: ./mvnw -version
+      - name: "Set RUN_ATTEMPT env variable"
+        run: echo "I_RUN_ATTEMPT=${{ github.run_attempt }}" >> $GITHUB_ENV
       # step 5
       - name: "Restore local maven repository cache"
         uses: actions/cache/restore@v4
@@ -99,6 +101,15 @@ jobs:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ env.TODAY }}
       # step 8
+      - name: "Upload Surefire reports"
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: run-${{ github.run_attempt }}-surefire-reports-${{ matrix.java }}
+          path: |
+            **/target/surefire-reports/*.xml
+            **/target/failsafe-reports/*.xml
+      # step 9
       - name: "Codecov"
         if: matrix.java == '8'
         uses: codecov/[email protected]
@@ -139,4 +150,4 @@ jobs:
                        mvn clean install \
                            -Prelease-seata \
                            -DskipTests \
-                           -e -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'
+                           -e -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'

.github/workflows/detect-flaky-test.yml

@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Detect flaky Test
+
+on:
+  workflow_run:
+    workflows: ["build"]
+    types:
+      - completed
+
+permissions:
+  issues: write
+
+jobs:
+  detect-flaky:
+    if: github.event.workflow_run.conclusion == 'success' && fromJSON(github.event.workflow_run.run_attempt) == 2
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [ 8, 17, 21 ]
+    steps:
+      # step 1
+      - name: Checkout source code
+        uses: actions/checkout@v4
+      # step 2
+      - name: Download artifacts for both runs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          JAVA_VERSION: ${{ matrix.java }}
+        run: |
+          download_artifact() { 
+            ARTIFACT_NAME=$1
+            OUTPUT_DIR=$2
+
+            echo "Fetching artifacts for workflow run ID: $RUN_ID"
+            artifacts_url="https://api.github.com/repos/${{ github.repository }}/actions/runs/$RUN_ID/artifacts"
+
+            artifact_url=$(curl -s -H "Authorization: token $GITHUB_TOKEN" $artifacts_url \
+              | jq -r --arg name "$ARTIFACT_NAME" '.artifacts[] | select(.name == $name) | .archive_download_url')
+
+            if [ -z "$artifact_url" ]; then
+              echo "❌ Artifact not found: $ARTIFACT_NAME"
+              exit 1
+            fi
+
+            echo "✅ Found artifact: $ARTIFACT_NAME"
+            curl -L -H "Authorization: token $GITHUB_TOKEN" "$artifact_url" -o artifact.zip
+            mkdir -p "$OUTPUT_DIR"
+            unzip artifact.zip -d "$OUTPUT_DIR"
+          }
+
+          download_artifact "run-1-surefire-reports-$JAVA_VERSION" "reports-1-$JAVA_VERSION"
+          download_artifact "run-2-surefire-reports-$JAVA_VERSION" "reports-2-$JAVA_VERSION"
+      # step 3
+      - name: "Use Python 3.x"
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      # step 4
+      - name: Parse failed tests
+        id: parse_failed_tests
+        run: |
+          output=$(python3 .github/scripts/parse_failed_tests.py reports-1-${{ matrix.java }} reports-2-${{ matrix.java }})
+          echo "flaky_tests=$output" >> "$GITHUB_OUTPUT"
+      - name: Create GitHub issue for flaky tests
+        if: steps.parse_failed_tests.outputs.flaky_tests != '[]'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const flakyTests = JSON.parse(process.env.FLAKY_TESTS);
+            const body = `Detected flaky tests in Java ${{ matrix.java }} run:\n\n` +
+              flakyTests.map(t => `- ${t}`).join('\n');
+            await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `Flaky tests detected for Java ${{ matrix.java }}`,
+              body: body,
+              labels: ['flaky-test'],
+            });
+        env:
+          FLAKY_TESTS: ${{ steps.parse_failed_tests.outputs.flaky_tests }}

changes/en-us/2.x.md

@@ -35,6 +35,8 @@ Add changes here for all PR submitted to the 2.x branch.
 ### optimize:
 
 - [[#7557](https://github.com/seata/seata/pull/7557)] upgrade some npmjs dependencies
+- [[#7545](https://github.com/apache/incubator-seata/pull/7545)] optimize CI configuration to detect flaky tests
+
 
 ### security:
 

changes/zh-cn/2.x.md

@@ -36,6 +36,8 @@
 ### optimize:
 
 - [[#7557](https://github.com/seata/seata/pull/7557)] 升级 npmjs 依赖
+- [[#7545](https://github.com/apache/incubator-seata/pull/7545)] 优化 CI 配置以检测 flaky-test
+
 
 ### security: