[QA] VizroAI UI tests

.github/actions/failed-artifacts-and-slack-notifications/action.yml

@@ -7,16 +7,16 @@ runs:
     - name: Copy failed screenshots
       shell: bash
       run: |
-        mkdir /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/
-        cd /home/runner/work/vizro/vizro/vizro-core/
+        mkdir ${{ env.PROJECT_PATH }}failed_screenshots/
+        cd ${{ env.PROJECT_PATH }}
         cp *.png failed_screenshots
 
     - name: Archive production artifacts
       uses: actions/upload-artifact@v4
       with:
         name: Failed screenshots
         path: |
-          /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/*.png
+          ${{ env.PROJECT_PATH }}failed_screenshots/*.png
 
     - name: Send custom JSON data to Slack
       id: slack

.github/workflows/test-e2e-component-library-vizro-core.yml

@@ -43,3 +43,4 @@ jobs:
         env:
           TESTS_NAME: Vizro e2e component library tests
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          PROJECT_PATH: /home/runner/work/vizro/vizro/vizro-core/

.github/workflows/test-e2e-vizro-ai-ui.yml

@@ -0,0 +1,66 @@
+name: e2e tests for VizroAI UI
+
+defaults:
+  run:
+    working-directory: vizro-ai
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "vizro-ai/**"
+      - "!vizro-ai/docs/**"
+
+env:
+  PYTHONUNBUFFERED: 1
+  FORCE_COLOR: 1
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  test-e2e-vizro-ai-ui-fork:
+    if: ${{ github.event.pull_request.head.repo.fork }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Passed fork step
+        run: echo "Success!"
+
+  test-e2e-vizro-ai-ui:
+    if: ${{ ! github.event.pull_request.head.repo.fork }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Hatch
+        run: pip install hatch
+
+      - name: Show dependency tree
+        run: hatch run pip tree
+
+      - name: Run e2e VizroAI UI tests
+        run: |
+          hatch run vizro-ai-ui
+          tests/tests_utils/wait-for-it.sh 127.0.0.1:8050 -t 30
+          hatch run test-e2e-vizro-ai-ui
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+
+      - name: Create artifacts and slack notifications
+        if: failure()
+        uses: ./.github/actions/failed-artifacts-and-slack-notifications
+        env:
+          TESTS_NAME: e2e VizroAI UI tests
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          PROJECT_PATH: /home/runner/work/vizro/vizro/vizro-ai/

.github/workflows/vizro-qa-tests-trigger.yml

@@ -20,7 +20,6 @@ jobs:
       matrix:
         include:
           - label: integration tests
-          - label: vizro-ai ui tests
     steps:
       - name: Passed fork step
         run: echo "Success!"
@@ -34,7 +33,6 @@ jobs:
       matrix:
         include:
           - label: integration tests
-          - label: vizro-ai ui tests
     steps:
       - uses: actions/checkout@v4
       - name: Tests trigger
@@ -44,8 +42,6 @@ jobs:
 
           if [ "${{ matrix.label }}" == "integration tests" ]; then
             export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_INTEGRATION_TESTS_WORKFLOW }}
-          elif [ "${{ matrix.label }}" == "vizro-ai ui tests" ]; then
-            export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_VIZRO_AI_UI_TESTS_WORKFLOW }}
           fi
           export INPUT_GITHUB_TOKEN=${{ secrets.VIZRO_SVC_PAT }}
           export INPUT_REF=main # because we should send existent branch to dispatch workflow

vizro-ai/changelog.d/20241118_131003_alexey_snigir_vizro_ai_ui_tests.md

@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+-->
+
+<!--
+### Highlights ✨
+
+- A bullet item for the Highlights ✨ category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Added
+
+- A bullet item for the Added category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->

vizro-ai/hatch.toml

@@ -48,6 +48,7 @@ prep-release = [
 ]
 pypath = "hatch run python -c 'import sys; print(sys.executable)'"
 test = "pytest tests {args}"
+test-e2e-vizro-ai-ui = "pytest -vs --reruns 1 tests/e2e/test_vizro_ai_ui.py --headless {args}"
 test-integration = "pytest -vs --reruns 1 tests/integration --headless {args}"
 test-score = "pytest -vs --reruns 1 tests/score --headless {args}"
 test-unit = "pytest tests/unit {args}"
@@ -56,6 +57,7 @@ test-unit-coverage = [
   "- coverage combine",
   "coverage report"
 ]
+vizro-ai-ui = "python examples/dashboard_ui/app.py &"
 
 [envs.docs]
 dependencies = [

vizro-ai/pyproject.toml

@@ -66,8 +66,12 @@ filterwarnings = [
   # Ignore LLMchian deprecation warning:
   "ignore:.*The class `LLMChain` was deprecated in LangChain 0.1.17",
   # Ignore warning for Pydantic v1 API and Python 3.13:
-  "ignore:Failing to pass a value to the 'type_params' parameter of 'typing.ForwardRef._evaluate' is deprecated:DeprecationWarning"
+  "ignore:Failing to pass a value to the 'type_params' parameter of 'typing.ForwardRef._evaluate' is deprecated:DeprecationWarning",
+  # Ignore deprecation warning until this is solved: https://github.com/plotly/dash/issues/2590:
+  "ignore:HTTPResponse.getheader():DeprecationWarning"
 ]
+norecursedirs = ["tests/tests_utils"]
+pythonpath = ["tests/tests_utils"]
 
 [tool.ruff]
 extend = "../pyproject.toml"

vizro-ai/tests/e2e/conftest.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+
+import pytest
+from e2e_asserts import browser_console_warnings_checker
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+
+@pytest.fixture()
+def chromedriver(request):
+    """Fixture for starting chromedriver."""
+    options = Options()
+    options.add_argument("--headless")
+    options.add_argument("--window-size=1920,1080")
+    options.add_argument("--disable-search-engine-choice-screen")
+    driver = webdriver.Chrome(options=options)
+    driver.get(f"http://127.0.0.1:{request.param.get('port')}/")
+    return driver
+
+
+@pytest.fixture(autouse=True)
+def teardown_method(chromedriver):
+    """Fixture checks log errors and quits the driver after each test."""
+    yield
+    log_levels = [level for level in chromedriver.get_log("browser") if level["level"] == "SEVERE" or "WARNING"]
+    if log_levels:
+        for log_level in log_levels:
+            browser_console_warnings_checker(log_level, log_levels)
+    chromedriver.quit()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    rep = outcome.get_result()
+    setattr(item, "rep_" + rep.when, rep)
+
+
+@pytest.fixture(scope="function", autouse=True)
+def test_failed_check(request):
+    yield
+    if request.node.rep_setup.failed:
+        return "setting up a test failed!", request.node.nodeid
+    elif request.node.rep_setup.passed and request.node.rep_call.failed:
+        driver = request.node.funcargs["chromedriver"]
+        take_screenshot(driver, request.node.nodeid)
+        return "executing test failed", request.node.nodeid
+
+
+def take_screenshot(driver, nodeid):
+    file_name = f'{nodeid}_{datetime.today().strftime("%Y-%m-%d_%H-%M")}.png'.replace("/", "_").replace("::", "__")
+    driver.save_screenshot(file_name)

vizro-ai/tests/e2e/test_vizro_ai_ui.py

@@ -0,0 +1,62 @@
+import os
+
+import pytest
+from e2e_fake_data_generator import create_genre_popularity_by_country
+from e2e_waiters import (
+    wait_for,
+    webdriver_click_waiter,
+    webdriver_waiter,
+    webdriver_waiter_css,
+)
+from selenium.common import InvalidSelectorException, TimeoutException
+
+
+@pytest.mark.parametrize(
+    "chromedriver",
+    [({"port": 8050})],
+    indirect=["chromedriver"],
+)
+def test_chart_ui(chromedriver):
+    # Create test dataset
+    popularity_dataset = create_genre_popularity_by_country(start_year=1980, end_year=2023, records_per_year=10)
+    # Save to a CSV file
+    popularity_dataset.to_csv("tests/tests_utils/genre_popularity_by_country.csv", index=False)
+
+    # fill in values
+    api_key = webdriver_waiter(chromedriver, '//*[@id="settings-api-key"]')
+    api_base = webdriver_waiter(chromedriver, '//*[@id="settings-api-base"]')
+    api_key.send_keys(os.environ["OPENAI_API_KEY"])
+    api_base.send_keys(os.environ["OPENAI_API_BASE"])
+
+    # close panel
+    webdriver_click_waiter(chromedriver, '//*[@class="btn-close"]')
+
+    # upload file
+    file_input = webdriver_waiter_css(chromedriver, 'input[type="file"]')
+    file_input.send_keys(os.path.abspath("tests/tests_utils/genre_popularity_by_country.csv"))
+    webdriver_click_waiter(chromedriver, '//*[@id="data-upload"]')
+
+    # enter prompt
+    prompt = webdriver_waiter(chromedriver, '//*[@id="text-area"]')
+    prompt.send_keys("Create bar graph by genre")
+
+    # choose gpt version
+    webdriver_click_waiter(chromedriver, '//*[@class="Select-arrow"]')
+    webdriver_waiter(chromedriver, '//*[@class="Select-menu-outer"]')
+    webdriver_click_waiter(chromedriver, '//*/div[text()="gpt-4o-mini"]')
+
+    # click run VizroAI
+    webdriver_click_waiter(chromedriver, '//*[@id="trigger-button"]')
+
+    # check result
+    def _text_waiter():
+        try:
+            webdriver_waiter(
+                chromedriver,
+                '//*[starts-with(@class, "language-python")]',
+            )
+            return True
+        except (TimeoutException, InvalidSelectorException):
+            return False
+
+    wait_for(_text_waiter)

vizro-ai/tests/tests_utils/e2e_asserts.py

@@ -0,0 +1,30 @@
+from e2e_constants import (
+    INVALID_PROP_ERROR,
+    REACT_NOT_RECOGNIZE_ERROR,
+    REACT_RENDERING_ERROR,
+    READPIXELS_WARNING,
+    SCROLL_ZOOM_ERROR,
+    UNMOUNT_COMPONENTS_ERROR,
+    WEBGL_WARNING,
+    WILLMOUNT_RENAMED_WARNING,
+    WILLRECEIVEPROPS_RENAMED_WARNING,
+)
+from hamcrest import any_of, assert_that, contains_string
+
+
+def browser_console_warnings_checker(log_level, log_levels):
+    assert_that(
+        log_level["message"],
+        any_of(
+            contains_string(INVALID_PROP_ERROR),
+            contains_string(REACT_NOT_RECOGNIZE_ERROR),
+            contains_string(SCROLL_ZOOM_ERROR),
+            contains_string(REACT_RENDERING_ERROR),
+            contains_string(UNMOUNT_COMPONENTS_ERROR),
+            contains_string(WILLMOUNT_RENAMED_WARNING),
+            contains_string(WILLRECEIVEPROPS_RENAMED_WARNING),
+            contains_string(READPIXELS_WARNING),
+            contains_string(WEBGL_WARNING),
+        ),
+        reason=f"Error outoput: {log_levels}",
+    )

vizro-ai/tests/tests_utils/e2e_constants.py

@@ -0,0 +1,11 @@
+INVALID_PROP_ERROR = "Invalid prop `persisted_props[0]` of value `on` supplied to `t`"
+REACT_NOT_RECOGNIZE_ERROR = "React does not recognize the `%s` prop on a DOM element"
+SCROLL_ZOOM_ERROR = "_scrollZoom"
+REACT_RENDERING_ERROR = "unstable_flushDiscreteUpdates: Cannot flush updates when React is already rendering"
+UNMOUNT_COMPONENTS_ERROR = "React state update on an unmounted component"
+WILLMOUNT_RENAMED_WARNING = "componentWillMount has been renamed"
+WILLRECEIVEPROPS_RENAMED_WARNING = "componentWillReceiveProps has been renamed"
+READPIXELS_WARNING = "GPU stall due to ReadPixels"
+WEBGL_WARNING = "WebGL"  # https://issues.chromium.org/issues/40277080
+
+TIMEOUT = 30