Another attempt to stabilize (and limit) integration tests (#39913)

There is no particular reason we should run the integration tests
for both Postgres and mysql - and Postgres ones seem to be much more
stable in general, so we only limit the tests to be run the tests for
Postgres.

Also - since the tests (especially mssql) seem to be flaky, we
re-run them once if they fail

Author: potiuk

.github/workflows/integration-tests.yml

@@ -59,20 +59,18 @@ on:  # yamllint disable-line rule:truthy
 jobs:
   tests-integration:
     timeout-minutes: 130
-    name: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}
+    name: "Integration Tests: ${{ matrix.integration }}"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     strategy:
       fail-fast: false
       matrix:
-        backend: ["postgres", "mysql"]
         integration: ${{ fromJSON(inputs.testable-integrations) }}
     env:
       IMAGE_TAG: "${{ inputs.image-tag }}"
-      BACKEND: "${{ matrix.backend }}"
-      # yamllint disable-line rule:line-length
-      BACKEND_VERSION: ${{ matrix.backend == 'postgres' && inputs.default-postgres-version || inputs.default-mysql-version }}"
+      BACKEND: "postgres"
+      BACKEND_VERSION: ${{ inputs.default-postgres-version }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      JOB_ID: "integration-${{ matrix.backend }}-${{ matrix.integration }}"
+      JOB_ID: "integration-${{ matrix.integration }}"
       SKIP_PROVIDER_TESTS: "${{ inputs.skip-provider-tests }}"
       ENABLE_COVERAGE: "${{ inputs.run-coverage}}"
       DEBUG_RESOURCES: "${{ inputs.debug-resources }}"
@@ -92,13 +90,10 @@ jobs:
         run: ./scripts/ci/cleanup_docker.sh
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
-      - name: "Integration Tests  ${{ matrix.backend }}:${{ matrix.integration }}"
-        run: |
-          breeze down
-          breeze testing integration-tests --integration ${{ matrix.integration }}
-          breeze down
-      - name: "Post Tests success: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}"
+      - name: "Integration Tests: ${{ matrix.integration }}"
+        run: ./scripts/ci/testing/run_integration_tests_with_retry.sh ${{ matrix.integration }}
+      - name: "Post Tests success: Integration Tests ${{ matrix.integration }}"
         uses: ./.github/actions/post_tests_success
-      - name: "Post Tests failure: Integration Tests ${{ matrix.backend }}-${{ matrix.integration }}"
+      - name: "Post Tests failure: Integration Tests ${{ matrix.integration }}"
         uses: ./.github/actions/post_tests_failure
         if: failure()

scripts/ci/testing/run_integration_tests_with_retry.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export COLOR_RED=$'\e[31m'
+export COLOR_YELLOW=$'\e[33m'
+export COLOR_RESET=$'\e[0m'
+
+if [[ ! "$#" -eq 1 ]]; then
+    echo "${COLOR_RED}You must provide exactly one argument!.${COLOR_RESET}"
+    exit 1
+fi
+
+INTEGRATION=${1}
+
+breeze down
+set +e
+breeze testing integration-tests --integration "${INTEGRATION}"
+RESULT=$?
+set -e
+if [[ ${RESULT} != "0" ]]; then
+    echo
+    echo "${COLOR_YELLOW}Integration Tests failed. Retrying once${COLOR_RESET}"
+    echo
+    echo "This could be due to a flaky test, re-running once to re-check it After restarting docker."
+    echo
+    sudo service docker restart
+    breeze down
+    set +e
+    breeze testing integration-tests --integration "${INTEGRATION}"
+    RESULT=$?
+    set -e
+    if [[ ${RESULT} != "0" ]]; then
+        echo
+        echo "${COLOR_RED}The integration tests failed for the second time! Giving up${COLOR_RESET}"
+        echo
+        exit ${RESULT}
+    fi
+fi

tests/integration/cli/commands/test_celery_command.py

@@ -29,7 +29,7 @@
 
 
 @pytest.mark.integration("celery")
-@pytest.mark.backend("mysql", "postgres")
+@pytest.mark.backend("postgres")
 class TestWorkerServeLogs:
     @classmethod
     def setup_class(cls):

tests/integration/executors/test_celery_executor.py

@@ -97,7 +97,7 @@ def _prepare_app(broker_url=None, execute=None):
 
 
 @pytest.mark.integration("celery")
-@pytest.mark.backend("mysql", "postgres")
+@pytest.mark.backend("postgres")
 class TestCeleryExecutor:
     def setup_method(self) -> None:
         db.clear_db_runs()
@@ -276,7 +276,7 @@ def __ne__(self, other):
 
 
 @pytest.mark.integration("celery")
-@pytest.mark.backend("mysql", "postgres")
+@pytest.mark.backend("postgres")
 class TestBulkStateFetcher:
     bulk_state_fetcher_logger = "airflow.providers.celery.executors.celery_executor_utils.BulkStateFetcher"