test: flakiness testing workflow (#128)

Signed-off-by: Nicklas Lundin <[email protected]>

Author: nicklasl

.github/workflows/manual-tests.yaml

@@ -0,0 +1,31 @@
+name: Run Tests Manually
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to run the tests on'
+        required: true
+        default: 'main'
+      number_of_runs:
+        description: 'Number of runs for the script'
+        required: true
+        default: '10'
+      gradle_task:
+        description: 'Gradle task to execute'
+        required: false
+        default: 'testDebugUnitTest'
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Run the tests script
+        run: |
+          chmod +x ./run-tests.sh
+          ./run-tests.sh ${{ github.event.inputs.number_of_runs }} ${{ github.event.inputs.gradle_task }}

run-tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+# Create output directory for reports
+OUTPUT_DIR="test-reports"
+rm -rf "$OUTPUT_DIR"
+mkdir -p "$OUTPUT_DIR"
+
+# First arg is total runs
+if [ "$1" != "" ]; then
+    total_runs=$1
+else
+    total_runs=10
+fi
+
+# Second arg is the gradle task to run
+if [ "$2" != "" ]; then
+    gradle_task=$2
+else
+    gradle_task="testDebugUnitTest"
+fi
+
+success_count=0
+failure_count=0
+
+echo "Running test $total_runs times with gradle task $gradle_task..."
+echo "------------------------"
+start_time=$(date +%s)
+
+for ((i=1; i<=total_runs; i++)); do
+    echo -n "Run #$i: "
+    
+    # Run the specific test with correct flags
+    touch "$OUTPUT_DIR/test-run-$i.txt"
+    ./gradlew $gradle_task --rerun-tasks --daemon > "$OUTPUT_DIR/test-run-$i.txt" 2>&1
+    
+    if [ $? -eq 0 ]; then
+        ((success_count++))
+        echo -e "${GREEN}PASSED${NC}"
+    else
+        ((failure_count++))
+        echo -e "${RED}FAILED${NC}"
+        # Print the output of the test run
+        echo "Output of test run $i:"
+        cat "$OUTPUT_DIR/test-run-$i.txt"
+        # Copy the HTML report with run number
+        cp -R "./android/build/reports/tests/" "$OUTPUT_DIR/failure-run-$i/"
+    fi
+done
+
+echo "------------------------"
+echo "Execution time: $(($(date +%s) - start_time)) seconds"
+echo "Results:"
+echo "Total runs: $total_runs"
+echo -e "Successes: ${GREEN}$success_count${NC}"
+echo -e "Failures: ${RED}$failure_count${NC}"
+echo "Failure rate: $(( (failure_count * 100) / total_runs ))%"
+
+if [ $failure_count -gt 0 ]; then
+    echo "Failure reports saved in $OUTPUT_DIR/"
+fi