[APM] Migrate APM Cypress tests to on_merge from `on_merge_unsuppor…

…ted_ftrs` (elastic#203991)

## Summary

Closes elastic#203837
[Internal] Closes
elastic/observability-dev#4126

This PR moves APM Cypress tests to be run on the main pipeline instead
of the unsupported one.

---------

Co-authored-by: kibanamachine <[email protected]>

.buildkite/ftr_oblt_stateful_configs.yml

@@ -1,10 +1,9 @@
 disabled:
   # Cypress configs, for now these are still run manually
+  - x-pack/test/apm_cypress/cli_config.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
   - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
   - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts

.buildkite/pipelines/on_merge.yml

@@ -479,6 +479,22 @@ steps:
         - exit_status: '-1'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 3
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
   - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
     label: Trigger unsupported ftr tests
     timeout_in_minutes: 10

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -28,25 +28,6 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
-    label: 'APM Cypress Tests'
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    depends_on: build
-    env:
-      PING_SLACK_TEAM: "@obs-ux-infra_services-team"
-    timeout_in_minutes: 120
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
     label: 'Profiling Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -13,7 +13,7 @@ steps:
       - check_types
       - check_oas_snapshot
     timeout_in_minutes: 120
-    parallelism: 1 # TODO: Set parallelism when apm_cypress handles it
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/scripts/steps/functional/apm_cypress.sh

@@ -2,35 +2,15 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common.sh
 
-APM_CYPRESS_RECORD_KEY="$(vault_get apm-cypress-dashboard-record-key CYPRESS_RECORD_KEY)"
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
-.buildkite/scripts/copy_es_snapshot_cache.sh
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 export JOB=kibana-apm-cypress
-IS_FLAKY_TEST_RUNNER=${CLI_COUNT:-0}
-GH_APM_TEAM_LABEL="Team:APM"
-
-if (! is_pr); then
-  echo "--- Add GH labels to buildkite metadata"
-  ts-node .buildkite/scripts/steps/add_gh_labels_to_bk_metadata.ts BUILDKITE_MESSAGE true
-  GH_ON_MERGE_LABELS="$(buildkite-agent meta-data get gh_labels --default '')"
-fi
-
-# Enabling cypress dashboard recording when PR is labeled with `apm:cypress-record` and we are not using the flaky test runner OR on merge with Team:APM label applied
-if ([[ "$IS_FLAKY_TEST_RUNNER" -ne 1 ]] && is_pr_with_label "apm:cypress-record") || ([[ $GH_ON_MERGE_LABELS == *"$GH_APM_TEAM_LABEL"* ]]); then
-  CYPRESS_ARGS="--record --key "$APM_CYPRESS_RECORD_KEY" --parallel --ci-build-id "${BUILDKITE_BUILD_ID}""
-else
-  CYPRESS_ARGS=""
-fi
 
 echo "--- APM Cypress Tests"
 
-cd "$XPACK_DIR"
+cd "$XPACK_DIR/solutions/observability/plugins/apm/ftr_e2e"
 
-node solutions/observability/plugins/apm/scripts/test/e2e.js \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  $CYPRESS_ARGS
+set +e
+yarn cypress:run; status=$?; yarn junit:merge || :; exit $status

.github/CODEOWNERS

@@ -1390,6 +1390,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /test/api_integration/apis/ui_metric/*.ts @elastic/obs-ux-infra_services-team
 /x-pack/test/functional/apps/apm/ @elastic/obs-ux-infra_services-team
 /x-pack/test/apm_api_integration/ @elastic/obs-ux-infra_services-team
+/x-pack/test/apm_cypress/ @elastic/obs-ux-infra_services-team
 /src/apm.js @elastic/kibana-core @vigneshshanmugam
 /src/platform/packages/shared/kbn-utility-types/src/dot.ts @dgieselaar
 /src/platform/packages/shared/kbn-utility-types/src/dot_test.ts @dgieselaar

x-pack/solutions/observability/plugins/apm/dev_docs/testing.md

@@ -136,35 +136,27 @@ node x-pack/solutions/observability/plugins/apm/scripts/test/dat --runner --stat
 
 The E2E tests are located in [`x-pack/solutions/observability/plugins/apm/ftr_e2e`](../ftr_e2e).
 
-When PR is labeled with `apm:cypress-record`, test runs are recorded to the [Cypress Dashboard](https://dashboard.cypress.io).
-
-Tests run on buildkite PR pipeline are parallelized (4 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`.
+Tests run on buildkite PR pipeline are parallelized (8 parallel jobs) and are orchestrated by the Cypress dashboard service. It can be configured in [.buildkite/pipelines/pull_request/apm_cypress.yml](https://github.com/elastic/kibana/blob/main/.buildkite/pipelines/pull_request/apm_cypress.yml) with the property `parallelism`.
 
 ```yml
     ...
     depends_on: build
-    parallelism: 4
+    parallelism: 8
     ...
 ```
 
 [Test tips and best practices](../ftr_e2e/README.md)
 
-#### Start test server
+#### Start Cypress dashboard
 
 ```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --server
+node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --headed
 ```
 
 #### Run tests
 
 ```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --open
-```
-
-### Run tests multiple times to check for flakiness
-
-```
-node x-pack/solutions/observability/plugins/apm/scripts/test/e2e --runner --times <NUMBER> [--spec <FILE_NAME>]
+node x-pack/solutions/observability/plugins/apm/scripts/test/e2e
 ```
 
 ### A11y checks

x-pack/solutions/observability/plugins/apm/ftr_e2e/README.md

@@ -1,6 +1,6 @@
 # APM E2E
 
-APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/integration`.
+APM uses [FTR](../../../../../../packages/kbn-test/README.mdx) (functional test runner) and [Cypress](https://www.cypress.io/) to run the e2e tests. The tests are located at `kibana/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/e2e`.
 
 ## Tips and best practices
 

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress.config.ts

@@ -9,28 +9,28 @@ import { defineCypressConfig } from '@kbn/cypress-config';
 import { setupNodeEvents } from './setup_cypress_node_events';
 
 export default defineCypressConfig({
-  projectId: 'omwh6f',
+  reporter: '../../../../../../node_modules/cypress-multi-reporters',
+  reporterOptions: {
+    configFile: './reporter_config.json',
+  },
   fileServerFolder: './cypress',
   fixturesFolder: './cypress/fixtures',
   screenshotsFolder: './cypress/screenshots',
   videosFolder: './cypress/videos',
-  requestTimeout: 10000,
-  responseTimeout: 40000,
-  defaultCommandTimeout: 30000,
+  defaultCommandTimeout: 60000,
   execTimeout: 120000,
   pageLoadTimeout: 120000,
   viewportHeight: 1800,
   viewportWidth: 1440,
-  video: true,
-  screenshotOnRunFailure: true,
-  retries: {
-    runMode: 1,
-  },
+  video: false,
+  screenshotOnRunFailure: false,
   e2e: {
     setupNodeEvents,
     baseUrl: 'http://localhost:5601',
     supportFile: './cypress/support/e2e.ts',
-    specPattern: './cypress/e2e/**/*.cy.{js,jsx,ts,tsx}',
+    specPattern: './cypress/e2e/**/*.cy.ts',
     experimentalMemoryManagement: true,
+    numTestsKeptInMemory: 3,
+    experimentalRunAllSpecs: true,
   },
 });

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/.gitignore

@@ -1,2 +1,3 @@
 /videos/*
 /screenshots/*
+/downloads/*

x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/support/e2e.ts

@@ -10,4 +10,3 @@ Cypress.on('uncaught:exception', (err, runnable) => {
 });
 
 import './commands';
-// import './output_command_timings';

x-pack/solutions/observability/plugins/apm/ftr_e2e/package.json

@@ -0,0 +1,14 @@
+{
+  "author": "Elastic",
+  "name": "@kbn/apm-ftr-e2e",
+  "version": "1.0.0",
+  "private": true,
+  "license": "Elastic License 2.0",
+  "scripts": {
+    "cypress": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../security/plugins/security_solution/scripts/start_cypress_parallel --config-file ../observability/plugins/apm/ftr_e2e/cypress.config.ts --ftr-config-file ../../../../../test/apm_cypress/cli_config",
+    "cypress:open": "yarn cypress open",
+    "cypress:run": "yarn cypress run",
+    "junit:merge": "../../../../../../node_modules/.bin/mochawesome-merge ../../../../../../target/kibana-apm/cypress/results/mochawesome*.json > ../../../../../../target/kibana-apm/cypress/results/output.json && ../../../../../../node_modules/.bin/marge ../../../../../../target/kibana-apm/cypress/results/output.json --reportDir ../../../../../../target/kibana-apm/cypress/results && yarn junit:transform && mkdir -p ../../../../../../target/junit && cp ../../../../../../target/kibana-apm/cypress/results/*.xml ../../../../../../target/junit/",
+    "junit:transform": "node ../../../../security/plugins/security_solution/scripts/junit_transformer --pathPattern '../../../../../../target/kibana-apm/cypress/results/*.xml' --rootDirectory ../../../../../../ --reportName 'APM Cypress' --writeInPlace"
+  }
+}

x-pack/solutions/observability/plugins/apm/ftr_e2e/reporter_config.json

@@ -0,0 +1,10 @@
+{
+  "reporterEnabled": "mochawesome, mocha-junit-reporter",
+  "reporterOptions": {
+    "html": false,
+    "json": true,
+    "mochaFile": "../../../../../../target/kibana-apm/cypress/results/TEST-apm-cypress-[hash].xml",
+    "overwrite": false,
+    "reportDir": "../../../../../../target/kibana-apm/cypress/results"
+  }
+}

x-pack/solutions/observability/plugins/apm/ftr_e2e/setup_cypress_node_events.ts

@@ -13,8 +13,6 @@ import {
 import { createEsClientForTesting } from '@kbn/test';
 // eslint-disable-next-line @kbn/imports/no_unresolvable_imports
 import { initPlugin } from '@frsource/cypress-plugin-visual-regression-diff/plugins';
-import del from 'del';
-import { some } from 'lodash';
 import { Readable } from 'stream';
 
 export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) {
@@ -75,18 +73,6 @@ export function setupNodeEvents(on: Cypress.PluginEvents, config: Cypress.Plugin
     },
   });
 
-  on('after:spec', (spec, results) => {
-    // Delete videos that have no failures or retries
-    if (results && results.video) {
-      const failures = some(results.tests, (test) => {
-        return some(test.attempts, { state: 'failed' });
-      });
-      if (!failures) {
-        del(results.video);
-      }
-    }
-  });
-
   on('before:browser:launch', (browser, launchOptions) => {
     if (browser.name === 'electron' && browser.isHeadless) {
       launchOptions.preferences.width = 1440;

x-pack/solutions/observability/plugins/apm/ftr_e2e/tsconfig.json

@@ -11,11 +11,8 @@
     "@kbn/test",
     "@kbn/apm-synthtrace",
     "@kbn/apm-synthtrace-client",
-    "@kbn/dev-utils",
     "@kbn/axe-config",
     "@kbn/cypress-config",
     "@kbn/apm-plugin",
-    "@kbn/ftr-common-functional-services",
-    "@kbn/ftr-common-functional-ui-services"
   ]
 }