Make tests and infrastructure less flaky (#273)

Author: NikolaSimsic

.github/actions/start-runtime/action.yml

@@ -14,40 +14,38 @@ runs:
       uses: actions/[email protected]
       with:
         python-version: "3.x"
+
     - name: "Install Python dependencies"
       run: pip install pyaml httplib2
       shell: bash
+
     - name: "Setup Java 21"
       id: setup-java
       uses: actions/setup-java@3a4f6e1af504cf6a31855fa899c6aa5355ba6c12 #v4
       with:
         distribution: "temurin"
         java-version: "21"
-    - name: "Extract deployment package"
-      run: |
-        mkdir project
-        unzip -qq ${{ inputs.mda-file }} -d project
-        cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
-        sed -i -- 's=$ROOT_PATH=${{ github.workspace }}=g' project/m2ee-native.yml
-        sed -i -- 's=$JAVA_HOME=${{ steps.setup-java.outputs.path }}=g' project/m2ee-native.yml
+    
+    - name: "Make setup-runtime.sh executable"
+      run: chmod +x .github/scripts/setup-runtime.sh
       shell: bash
-    - name: "Setup m2ee"
+
+    - name: "Initial setup"
       run: |
-        mkdir -p var/log var/opt/m2ee var/run bin tmp
-        git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
-        mv tmp/m2ee/src/* var/opt/m2ee
-        chmod a=rwx var/log/ var/run/
-        echo "#!/bin/bash -x" > bin/m2ee
-        echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
-        chmod +x bin/m2ee
+        .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
       shell: bash
-    - name: "Setup mxruntime"
+
+    - name: "Start mxruntime with retries"
       run: |
-        mkdir -p ${{ github.workspace }}/project/runtimes ${{ github.workspace }}/project/data/model-upload ${{ github.workspace }}/project/data/database ${{ github.workspace }}/project/data/files ${{ github.workspace }}/project/data/tmp
-        wget -q https://cdn.mendix.com/runtime/mendix-${{ inputs.mendix-version }}.tar.gz -O tmp/runtime.tar.gz
-        tar xfz tmp/runtime.tar.gz --directory ${{ github.workspace }}/project/runtimes
-        rm tmp/runtime.tar.gz
-      shell: bash
-    - name: "Start mxruntime"
-      run: bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start
-      shell: bash
+        MAX_RETRIES=3
+        RETRY=0
+        until bin/m2ee -c ${{ github.workspace }}/project/m2ee-native.yml --verbose --yolo start; do
+          RETRY=$((RETRY+1))
+          if [ $RETRY -ge $MAX_RETRIES ]; then
+            echo "mxruntime failed after $MAX_RETRIES attempts."
+            exit 1
+          fi
+          echo "mxruntime failed, retrying ($RETRY/$MAX_RETRIES)..."
+          .github/scripts/setup-runtime.sh "${{ inputs.mda-file }}" "${{ inputs.mendix-version }}" "${{ steps.setup-java.outputs.path }}" "${{ github.workspace }}"
+        done
+      shell: bash

.github/scripts/determine-nt-version.py

@@ -37,51 +37,48 @@
 if best_match:
     print(f"Best matching range: {best_match}")
 
-    # Get the major version to look for in tags
+    # Get the major version to look for in releases
     max_pattern = version_data[best_match].get("max", "")
-    major_version = max_pattern.split('.')[0]
-    
-    print(f"Looking for latest version with major version: {major_version}")
+    if max_pattern == "*":
+        major_version = None
+        print("Looking for latest available release (no major version restriction)")
+    else:
+        major_version = max_pattern.split('.')[0]
+        print(f"Looking for latest release with major version: {major_version}")
 
-    # Get available tags from native-template repository
-    response = requests.get('https://api.github.com/repos/mendix/native-template/tags')
+    # Get available releases from native-template repository
+    response = requests.get('https://api.github.com/repos/mendix/native-template/releases')
     if response.status_code == 200:
-        all_tags = response.json()
-        tag_names = [tag['name'] for tag in all_tags]
-        print(f"Available tags: {tag_names}")
+        all_releases = response.json()
+        release_names = [release['tag_name'] for release in all_releases]
+        print(f"Available releases: {release_names}")
 
-        # Find the latest version matching the major version
-        matching_tags = []
+        # Find the latest release matching the major version
+        matching_releases = []
 
-        for tag in tag_names:
-            # Remove 'v' prefix if present for comparison
+        for release in all_releases:
+            tag = release['tag_name']
             clean_tag = tag[1:] if tag.startswith('v') else tag
-            
-            # Check if the tag starts with the major version
-            if clean_tag.startswith(f"{major_version}."):
-                try:
-                    # Try to parse as a version (this handles proper version numbers)
-                    tag_version = version.parse(clean_tag)
-                    matching_tags.append((tag, tag_version))
-                except:
-                    # Skip tags that don't parse as proper versions
-                    pass
+            try:
+                tag_version = version.parse(clean_tag)
+                if major_version is None or clean_tag.startswith(f"{major_version}."):
+                    matching_releases.append((tag, tag_version))
+            except:
+                pass
 
-        if matching_tags:
-            # Sort by version (highest last) and get the last one
-            matching_tags.sort(key=lambda x: x[1])
-            latest_version = matching_tags[-1][0]
-            print(f"Selected Native Template version: {latest_version}")
+        if matching_releases:
+            matching_releases.sort(key=lambda x: x[1])
+            latest_tag = matching_releases[-1][0]
+            print(f"Selected Native Template release: {latest_tag}")
             with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
-                f.write(f"nt_branch={latest_version}\n")
+                f.write(f"nt_branch={latest_tag}\n")
         else:
-            # Fallback to min version if no matching tag found
             min_version = version_data[best_match].get("min", "")
-            print(f"No matching tag found, using minimum version: {min_version}")
+            print(f"No matching release found, using minimum version: {min_version}")
             with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
                 f.write(f"nt_branch={min_version}\n")
     else:
-        print(f"Failed to get tags: {response.status_code}")
+        print(f"Failed to get releases: {response.status_code}")
         print("Using master as fallback")
         with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
             f.write("nt_branch=master\n")

.github/scripts/setup-runtime.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+MDA_FILE="$1"
+MENDIX_VERSION="$2"
+JAVA_PATH="$3"
+WORKSPACE="$4"
+
+rm -rf project var tmp bin
+
+mkdir project
+unzip -qq "$MDA_FILE" -d project
+cp configs/e2e/m2ee-native.yml project/m2ee-native.yml
+sed -i -- "s=\$ROOT_PATH=$WORKSPACE=g" project/m2ee-native.yml
+sed -i -- "s=\$JAVA_HOME=$JAVA_PATH=g" project/m2ee-native.yml
+
+mkdir -p var/log var/opt/m2ee var/run bin tmp
+git clone https://github.com/KevinVlaanderen/m2ee-tools.git tmp/m2ee
+mv tmp/m2ee/src/* var/opt/m2ee
+chmod a=rwx var/log/ var/run/
+echo "#!/bin/bash -x" > bin/m2ee
+echo "python3 var/opt/m2ee/m2ee.py \$@" >>bin/m2ee
+chmod +x bin/m2ee
+
+mkdir -p "$WORKSPACE/project/runtimes" "$WORKSPACE/project/data/model-upload" "$WORKSPACE/project/data/database" "$WORKSPACE/project/data/files" "$WORKSPACE/project/data/tmp"
+wget -q "https://cdn.mendix.com/runtime/mendix-$MENDIX_VERSION.tar.gz" -O tmp/runtime.tar.gz
+tar xfz tmp/runtime.tar.gz --directory "$WORKSPACE/project/runtimes"
+rm tmp/runtime.tar.gz

.github/workflows/NativePipeline.yml

@@ -439,6 +439,7 @@ jobs:
   android-tests:
     needs: [scope, mendix-version, project, android-app]
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}
@@ -557,6 +558,7 @@ jobs:
   ios-tests:
     needs: [scope, mendix-version, project, ios-app]
     runs-on: macos-15
+    timeout-minutes: 60
     strategy:
       matrix:
         widget: ${{ fromJson(needs.scope.outputs.widgets) }}

configs/e2e/mendix-versions.json

@@ -1,4 +1,3 @@
 {
-    "latest": "10.22.0.68245",
-    "8": "8.18.23.62193"
+    "latest": "10.24.0.73019"
 }

maestro/helpers/compare_screenshots.js

@@ -57,7 +57,8 @@ fs.readdirSync(actualDir).forEach(file => {
       ];
     } else if (platform === 'android') {
       ignoredAreas = [
-        { x: 0, y: 0, width, height: 50 } // Ignore top 40 pixels on Android
+        { x: 0, y: 0, width, height: 50 }, // Ignore top 50 pixels on Android
+        { x: width - 15, y: 0, width: 15, height } // Ignore right 15 pixels on Android
       ];
     }