[ML] Add AD job validation UI tests

x-pack/plugins/ml/public/application/components/callout/callout.tsx

@@ -55,7 +55,7 @@ const Message: FC<Pick<CalloutMessage, 'text' | 'url'>> = ({ text, url }) => (
 export const Callout: FC<CalloutMessage> = ({ heading, status, text, url }) => (
   <>
     <EuiCallOut
-      data-test-subj={'mlValidationCallout'}
+      data-test-subj={`mlValidationCallout-${statusToEuiColor(status)}-${text.substring(0, 30)}`}
       // @ts-ignore
       color={statusToEuiColor(status)}
       size="s"

x-pack/plugins/ml/public/application/jobs/new_job/common/components/time_range_picker.tsx

@@ -85,21 +85,23 @@ export const TimeRangePicker: FC<Props> = ({ setTimeRange, timeRange }) => {
             />
           }
           endDateControl={
-            <EuiDatePicker
-              selected={endMoment}
-              onChange={handleChangeEnd}
-              startDate={startMoment}
-              endDate={endMoment}
-              aria-label={i18n.translate(
-                'xpack.ml.newJob.wizard.timeRangeStep.timeRangePicker.endDateLabel',
-                {
-                  defaultMessage: 'End date',
-                }
-              )}
-              showTimeSelect
-              dateFormat={dateFormat}
-              minDate={startMoment}
-            />
+            <div data-test-subj="mlJobWizardDatePickerRangeEndDate">
+              <EuiDatePicker
+                selected={endMoment}
+                onChange={handleChangeEnd}
+                startDate={startMoment}
+                endDate={endMoment}
+                aria-label={i18n.translate(
+                  'xpack.ml.newJob.wizard.timeRangeStep.timeRangePicker.endDateLabel',
+                  {
+                    defaultMessage: 'End date',
+                  }
+                )}
+                showTimeSelect
+                dateFormat={dateFormat}
+                minDate={startMoment}
+              />
+            </div>
           }
         />
       </div>

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/job_details_step/components/advanced_section/components/annotations/annotations_switch.tsx

@@ -53,6 +53,7 @@ export const AnnotationsSwitch: FC = () => {
       </Description>
       {showCallOut && (
         <EuiCallOut
+          data-test-subj="mlJobWizardAlsoEnableAnnotationsRecommendationCallout"
           title={
             <FormattedMessage
               id="xpack.ml.newJob.wizard.jobDetailsStep.advancedSection.annotationsSwitchCallout.title"

x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard_horizontal_steps.tsx

@@ -79,6 +79,10 @@ export const WizardHorizontalSteps: FC<Props> = ({
   }
 
   function createStepProps(step: WIZARD_STEPS) {
+    let testSubj: string = '';
+    if (step === WIZARD_STEPS.TIME_RANGE) testSubj = 'mlJobWizardTimeRangeStep';
+    if (step === WIZARD_STEPS.JOB_DETAILS) testSubj = 'mlJobWizardJobDetailsStep';
+
     return {
       onClick: () => jumpToStep(step),
       status: (currentStep === step
@@ -87,6 +91,7 @@ export const WizardHorizontalSteps: FC<Props> = ({
         ? 'complete'
         : 'incomplete') as EuiStepStatus,
       disabled: disableSteps || highestStep < step,
+      'data-test-subj': testSubj,
     };
   }
 

x-pack/test/functional/apps/ml/anomaly_detection_jobs/index.ts

@@ -40,6 +40,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./single_metric_job'));
 
     if (!isCcs) {
+      loadTestFile(require.resolve('./job_wizard_validation'));
       loadTestFile(require.resolve('./single_metric_job_without_datafeed_start'));
       loadTestFile(require.resolve('./multi_metric_job'));
       loadTestFile(require.resolve('./population_job'));

x-pack/test/functional/apps/ml/anomaly_detection_jobs/job_wizard_validation.ts

@@ -0,0 +1,171 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FtrProviderContext } from '../../../ftr_provider_context';
+import type { FieldStatsType } from '../common/types';
+
+export default function ({ getService }: FtrProviderContext) {
+  const config = getService('config');
+  const esNode = config.get('esTestCluster.ccs')
+    ? getService('remoteEsArchiver' as 'esArchiver')
+    : getService('esArchiver');
+  const ml = getService('ml');
+
+  const jobId = `fq_single_1_${Date.now()}`;
+  const jobGroups = ['automated', 'farequote', 'single-metric'];
+  const aggAndFieldIdentifier = 'Mean(responsetime)';
+  const bucketSpan = '30m';
+
+  const remoteName = 'ftr-remote:';
+  const esIndexPatternName = 'ft_farequote';
+  const esIndexPatternString = config.get('esTestCluster.ccs')
+    ? remoteName + esIndexPatternName
+    : esIndexPatternName;
+
+  const fieldStatsEntries = [
+    {
+      fieldName: '@version.keyword',
+      type: 'keyword' as FieldStatsType,
+      expectedValues: ['1'],
+    },
+  ];
+
+  describe('job wizard validation', function () {
+    this.tags(['ml']);
+    before(async () => {
+      await esNode.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
+      await ml.testResources.createDataViewIfNeeded(esIndexPatternString, '@timestamp');
+      await ml.testResources.setKibanaTimeZoneToUTC();
+
+      await ml.securityUI.loginAsMlPowerUser();
+
+      await ml.testExecution.logTestStep('job creation loads the job management page');
+      await ml.navigation.navigateToMl();
+      await ml.navigation.navigateToJobManagement();
+
+      await ml.testExecution.logTestStep('job creation loads the new job source selection page');
+      await ml.jobManagement.navigateToNewJobSourceSelection();
+
+      await ml.testExecution.logTestStep('job creation loads the job type selection page');
+      await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(esIndexPatternString);
+
+      await ml.testExecution.logTestStep('job creation loads the single metric job wizard page');
+      await ml.jobTypeSelection.selectSingleMetricJob();
+      await ml.testExecution.logTestStep('job creation displays the time range step');
+      await ml.jobWizardCommon.assertTimeRangeSectionExists();
+
+      await ml.testExecution.logTestStep('job creation sets the time range');
+      await ml.jobWizardCommon.clickUseFullDataButton(
+        'Feb 7, 2016 @ 00:00:00.000',
+        'Feb 11, 2016 @ 23:59:54.000'
+      );
+
+      await ml.testExecution.logTestStep('job creation displays the event rate chart');
+      await ml.jobWizardCommon.assertEventRateChartExists();
+      await ml.jobWizardCommon.assertEventRateChartHasData();
+
+      await ml.testExecution.logTestStep('job creation displays the pick fields step');
+      await ml.jobWizardCommon.advanceToPickFieldsSection();
+
+      await ml.testExecution.logTestStep('job creation opens field stats flyout from agg input');
+      await ml.jobWizardCommon.assertAggAndFieldInputExists();
+      for (const { fieldName, type: fieldType, expectedValues } of fieldStatsEntries) {
+        await ml.jobWizardCommon.assertFieldStatFlyoutContentFromAggSelectionInputTrigger(
+          fieldName,
+          fieldType,
+          expectedValues
+        );
+      }
+
+      await ml.testExecution.logTestStep('job creation selects field and aggregation');
+      await ml.jobWizardCommon.selectAggAndField(aggAndFieldIdentifier, true);
+      await ml.jobWizardCommon.assertAnomalyChartExists('LINE');
+
+      await ml.testExecution.logTestStep('job creation inputs the bucket span');
+      await ml.jobWizardCommon.assertBucketSpanInputExists();
+      await ml.jobWizardCommon.setBucketSpan(bucketSpan);
+
+      await ml.testExecution.logTestStep('job creation displays the job details step');
+      await ml.jobWizardCommon.advanceToJobDetailsSection();
+
+      await ml.testExecution.logTestStep('job creation inputs the job id');
+      await ml.jobWizardCommon.assertJobIdInputExists();
+      await ml.jobWizardCommon.setJobId(jobId);
+
+      await ml.testExecution.logTestStep('job creation inputs job groups');
+      await ml.jobWizardCommon.assertJobGroupInputExists();
+      for (const jobGroup of jobGroups) {
+        await ml.jobWizardCommon.addJobGroup(jobGroup);
+      }
+      await ml.jobWizardCommon.assertJobGroupSelection(jobGroups);
+
+      await ml.testExecution.logTestStep('job creation opens the additional settings section');
+      await ml.jobWizardCommon.ensureAdditionalSettingsSectionOpen();
+
+      await ml.testExecution.logTestStep('job creation opens the advanced section');
+      await ml.jobWizardCommon.ensureAdvancedSectionOpen();
+
+      await ml.testExecution.logTestStep('job creation displays the model plot switch');
+      await ml.jobWizardCommon.assertModelPlotSwitchExists();
+
+      await ml.testExecution.logTestStep('job creation enables the dedicated index switch');
+      await ml.jobWizardCommon.assertDedicatedIndexSwitchExists();
+      await ml.jobWizardCommon.activateDedicatedIndexSwitch();
+
+      await ml.testExecution.logTestStep('job creation inputs the model memory limit');
+      await ml.jobWizardCommon.assertModelMemoryLimitInputExists();
+    });
+
+    after(async () => {
+      await ml.api.cleanMlIndices();
+      await ml.testResources.deleteDataViewByTitle(esIndexPatternString);
+    });
+
+    it('job creation and toggling model change annotation triggers enable annotation recommendation callout', async () => {
+      await ml.jobWizardCommon.togglingModelChangeAnnotationsShowsCalloutAndRemovesCallout();
+    });
+
+    it('too short of a job creation time range results in validation callouts', async () => {
+      await ml.jobWizardCommon.goBackToTimeRange();
+
+      const tooShort = 'Feb 7, 2016 @ 00:01:00.000';
+      await ml.jobWizardCommon.setEndDate(tooShort);
+
+      await ml.testExecution.logTestStep('job creation displays the validation step');
+      await ml.jobWizardCommon.goBackToJobDetailsSection();
+      await ml.jobWizardCommon.advanceToValidationSection();
+      await ml.jobWizardCommon.assertValidationCallouts([
+        'mlValidationCallout-warning-The selected or available time',
+        'mlValidationCallout-danger-The datafeed preview failed. T',
+        'mlValidationCallout-warning-Job validation has failed, but',
+      ]);
+
+      await ml.jobWizardCommon.goBackToTimeRange();
+      await ml.jobWizardCommon.clickUseFullDataButton(
+        'Feb 7, 2016 @ 00:00:00.000',
+        'Feb 11, 2016 @ 23:59:54.000'
+      );
+      await ml.jobWizardCommon.advanceToValidationSection();
+      await ml.jobWizardCommon.assertValidationCallouts([
+        'mlValidationCallout-success-Valid and long enough to model',
+        'mlValidationCallout-success-Valid and within the estimated',
+      ]);
+    });
+
+    it('job creation memory limit too large results in validation callout', async () => {
+      await ml.jobWizardCommon.goBackToJobDetailsSection();
+
+      const tooLarge = '100000000MB';
+      await ml.jobWizardCommon.setModelMemoryLimit(tooLarge);
+
+      await ml.jobWizardCommon.clickNextButton();
+      await ml.jobWizardCommon.assertValidationCallouts([
+        'mlValidationCallout-warning-Job will not be able to run in',
+      ]);
+    });
+  });
+}

x-pack/test/functional/services/ml/job_wizard_common.ts

@@ -619,5 +619,67 @@ export function MachineLearningJobWizardCommonProvider(
         await testSubjects.existOrFail(expectedSelector);
       });
     },
+
+    async getAnnotationSwitchCheckedState(): Promise<boolean> {
+      const subj = 'mlJobWizardSwitchAnnotations';
+      const isSelected = await testSubjects.getAttribute(subj, 'aria-checked');
+      return isSelected === 'true';
+    },
+
+    async getAnnotationsSwitchCheckedState(expectedValue: boolean) {
+      const actualCheckedState = await this.getAnnotationSwitchCheckedState();
+      expect(actualCheckedState).to.eql(
+        expectedValue,
+        `Expected annotations switch to be '${expectedValue ? 'enabled' : 'disabled'}' (got '${
+          actualCheckedState ? 'enabled' : 'disabled'
+        }')`
+      );
+    },
+
+    async toggleAnnotationSwitch(toggle: boolean) {
+      const subj = 'mlJobWizardSwitchAnnotations';
+      if ((await this.getAnnotationSwitchCheckedState()) !== toggle) {
+        await retry.tryForTime(5 * 1000, async () => {
+          await testSubjects.clickWhenNotDisabledWithoutRetry(subj);
+          await this.getAnnotationSwitchCheckedState();
+        });
+      }
+    },
+
+    async togglingModelChangeAnnotationsShowsCalloutAndRemovesCallout() {
+      await this.toggleAnnotationSwitch(false);
+      await testSubjects.existOrFail('mlJobWizardAlsoEnableAnnotationsRecommendationCallout', {
+        timeout: 3_000,
+      });
+      await this.toggleAnnotationSwitch(true);
+      await testSubjects.missingOrFail('mlJobWizardAlsoEnableAnnotationsRecommendationCallout', {
+        timeout: 3_000,
+      });
+    },
+
+    async goBackToTimeRange() {
+      await testSubjects.existOrFail('mlJobWizardTimeRangeStep', {
+        timeout: 3_000,
+      });
+      await testSubjects.click('mlJobWizardTimeRangeStep');
+    },
+
+    async setEndDate(input: string) {
+      await testSubjects.setValue('mlJobWizardDatePickerRangeEndDate', input);
+    },
+
+    async goBackToJobDetailsSection() {
+      await testSubjects.existOrFail('mlJobWizardJobDetailsStep', {
+        timeout: 3_000,
+      });
+      await testSubjects.click('mlJobWizardJobDetailsStep');
+    },
+
+    async assertValidationCallouts(expectedCallOutSelectors: string[]) {
+      for await (const sel of expectedCallOutSelectors)
+        await testSubjects.existOrFail(sel, {
+          timeout: 3_000,
+        });
+    },
   };
 }