From 62e89180127f05352918d7b5d0cb66c00497c248 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 28 Jul 2020 15:48:31 -0500
Subject: [PATCH 01/37] [ML] Add decision path charts

---
 .../ml/common/types/feature_importance.ts     |  10 ++
 .../components/data_grid/common.ts            |  15 +-
 .../components/data_grid/data_grid.tsx        |  51 ++++++-
 .../decision_path_chart.tsx                   | 130 ++++++++++++++++++
 .../decision_path_json_viewer.tsx             |  16 +++
 .../decision_path_popover.tsx                 |  64 +++++++++
 .../exploration_results_table.tsx             |   1 +
 .../use_exploration_results.ts                |   1 -
 .../ml_api_service/data_frame_analytics.ts    |   8 ++
 .../ml/server/routes/data_frame_analytics.ts  |  61 ++++++++
 .../routes/schemas/data_analytics_schema.ts   |   8 ++
 11 files changed, 352 insertions(+), 13 deletions(-)
 create mode 100644 x-pack/plugins/ml/common/types/feature_importance.ts
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx

diff --git a/x-pack/plugins/ml/common/types/feature_importance.ts b/x-pack/plugins/ml/common/types/feature_importance.ts
new file mode 100644
index 0000000000000..e93018253c38c
--- /dev/null
+++ b/x-pack/plugins/ml/common/types/feature_importance.ts
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface FeatureImportance {
+  feature_name: string;
+  importance: number;
+}
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
index 1f0fcb63f019d..434a443c003f9 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
@@ -5,7 +5,7 @@
  */
 
 import moment from 'moment-timezone';
-import { useEffect, useMemo } from 'react';
+import React, { useEffect, useMemo } from 'react';
 
 import {
   EuiDataGridCellValueElementProps,
@@ -119,13 +119,14 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
       schema = 'numeric';
     }
 
-    if (
-      field.includes(`${resultsField}.${FEATURE_IMPORTANCE}`) ||
-      field.includes(`${resultsField}.${TOP_CLASSES}`)
-    ) {
+    if (field.includes(`${resultsField}.${TOP_CLASSES}`)) {
       schema = 'json';
     }
 
+    if (field.includes(`${resultsField}.${FEATURE_IMPORTANCE}`)) {
+      schema = 'featureImportance';
+    }
+
     return { id: field, schema, isSortable };
   });
 };
@@ -250,10 +251,6 @@ export const useRenderCellValue = (
         return cellValue ? 'true' : 'false';
       }
 
-      if (typeof cellValue === 'object' && cellValue !== null) {
-        return JSON.stringify(cellValue);
-      }
-
       return cellValue;
     };
   }, [indexPattern?.fields, pagination.pageIndex, pagination.pageSize, tableItems]);
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index d4be2eab13d26..931c850d3357f 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -5,8 +5,7 @@
  */
 
 import { isEqual } from 'lodash';
-import React, { memo, useEffect, FC } from 'react';
-
+import React, { memo, useEffect, FC, useState, useCallback } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import {
@@ -24,13 +23,16 @@ import {
 } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
+import { useMlKibana } from '../../contexts/kibana/kibana_context';
 
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
 
-import { INDEX_STATUS } from '../../data_frame_analytics/common';
+import { DataFrameAnalyticsConfig, INDEX_STATUS } from '../../data_frame_analytics/common';
 
 import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
 import { UseIndexDataReturnType } from './types';
+import { DecisionPathPopover } from './feature_importance/decision_path_popover';
+
 // TODO Fix row hovering + bar highlighting
 // import { hoveredRow$ } from './column_chart';
 
@@ -41,6 +43,7 @@ export const DataGridTitle: FC<{ title: string }> = ({ title }) => (
 );
 
 interface PropsWithoutHeader extends UseIndexDataReturnType {
+  jobConfig?: DataFrameAnalyticsConfig;
   dataTestSubj: string;
   toastNotifications: CoreSetup['notifications']['toasts'];
 }
@@ -60,6 +63,7 @@ type Props = PropsWithHeader | PropsWithoutHeader;
 export const DataGrid: FC<Props> = memo(
   (props) => {
     const {
+      jobConfig,
       chartsVisible,
       chartsButtonVisible,
       columnsWithCharts,
@@ -81,6 +85,11 @@ export const DataGrid: FC<Props> = memo(
       toggleChartVisibility,
       visibleColumns,
     } = props;
+    const {
+      services: {
+        mlServices: { mlApiServices },
+      },
+    } = useMlKibana();
 
     // TODO Fix row hovering + bar highlighting
     // const getRowProps = (item: any) => {
@@ -90,6 +99,29 @@ export const DataGrid: FC<Props> = memo(
     //   };
     // };
 
+    const [baseline, setBaseLine] = useState();
+
+    const getAnalyticsBaseline = useCallback(async () => {
+      try {
+        const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
+          jobConfig.id,
+          jobConfig.dest.index,
+          jobConfig.analysis.classification?.prediction_field_name ??
+            jobConfig.analysis.regression?.prediction_field_name
+        );
+        if (result?.baseline) {
+          setBaseLine(result.baseline);
+        }
+      } catch (e) {
+        // eslint-disable-next-line
+        console.error(e);
+      }
+    }, [mlApiServices, jobConfig]);
+
+    useEffect(() => {
+      getAnalyticsBaseline();
+    }, [jobConfig]);
+
     useEffect(() => {
       if (invalidSortingColumnns.length > 0) {
         invalidSortingColumnns.forEach((columnId) => {
@@ -225,6 +257,19 @@ export const DataGrid: FC<Props> = memo(
                   }
                 : {}),
             }}
+            popoverContents={{
+              featureImportance: ({ cellContentsElement }) => {
+                const stringContents = cellContentsElement.textContent;
+                const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
+                return (
+                  <DecisionPathPopover
+                    analyticsId={jobConfig.id}
+                    baseline={baseline}
+                    featureImportance={parsedFIArray}
+                  />
+                );
+              },
+            }}
             pagination={{
               ...pagination,
               pageSizeOptions: [5, 10, 25],
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
new file mode 100644
index 0000000000000..7347cad554352
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+import {
+  Chart,
+  Datum,
+  Settings,
+  LineAnnotation,
+  AnnotationDomainTypes,
+  LineSeries,
+  Axis,
+  ScaleType,
+  Position,
+  LineAnnotationDatum,
+} from '@elastic/charts';
+import _ from 'lodash';
+import { i18n } from '@kbn/i18n';
+import { EuiIcon } from '@elastic/eui';
+
+const style = {
+  line: {
+    strokeWidth: 1,
+    stroke: '#ff0000',
+    opacity: 1,
+  },
+  details: {
+    fontSize: 12,
+    fontFamily: 'Arial',
+    fontStyle: 'bold',
+    fill: 'gray',
+    padding: 0,
+  },
+};
+
+interface FeatureImportance {
+  feature_name: string;
+  importance: number;
+}
+interface FeatureImportanceDecisionPathProps {
+  baseline?: number;
+  featureImportance: FeatureImportance[];
+}
+
+const FEATURE_NAME = 'feature_name';
+const FEATURE_IMPORTANCE = 'importance';
+
+export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProps> = ({
+  baseline,
+  featureImportance,
+}) => {
+  const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
+
+  let mappedFeatureImportance: Datum[] = featureImportance;
+
+  if (baseline) {
+    // also plot the baseline in the decision path visualization
+    mappedFeatureImportance.push({ [FEATURE_NAME]: 'baseline', [FEATURE_IMPORTANCE]: baseline });
+
+    // get the absolute difference of the importance value to the baseline for sorting
+    mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
+      ...d,
+      difference: Math.abs(d[FEATURE_IMPORTANCE] - baseline),
+    }));
+
+    // sort so decision path goes from bottom to top
+    mappedFeatureImportance = mappedFeatureImportance
+      .sort((a, b) => b.difference - a.difference)
+      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+  } else {
+    mappedFeatureImportance = mappedFeatureImportance.map((d) => [
+      d[FEATURE_NAME],
+      d[FEATURE_IMPORTANCE],
+    ]);
+  }
+
+  const maxDomain = _.maxBy(featureImportance, (d) => d[1]);
+  const minDomain = _.minBy(featureImportance, (d) => d[1]);
+
+  return (
+    <div style={{ width: 300, height: mappedFeatureImportance.length * 75 }}>
+      {baseline && (
+        <div>
+          {i18n.translate('xpack.ml.dataframe.analytics.explorationResults.decisionPathBaseline', {
+            defaultMessage: 'Baseline: {baseline}',
+            values: { baseline: baseline?.toFixed(3) },
+          })}
+        </div>
+      )}
+      <Chart className="story-chart">
+        <Settings rotation={90} />
+        {baseline && (
+          <LineAnnotation
+            id="dfa_baseline"
+            domainType={AnnotationDomainTypes.YDomain}
+            dataValues={baselineData}
+            style={style}
+            marker={<EuiIcon type={'annotation'} />}
+          />
+        )}
+
+        <Axis
+          id="bottom"
+          position={Position.Bottom}
+          showOverlappingTicks
+          domain={
+            minDomain && maxDomain
+              ? {
+                  min: minDomain.importance,
+                  max: maxDomain.importance,
+                }
+              : undefined
+          }
+        />
+        <Axis id="left" position={Position.Left} />
+        <LineSeries
+          id="importance"
+          xScaleType={ScaleType.Ordinal}
+          yScaleType={ScaleType.Linear}
+          xAccessor={0}
+          yAccessors={[1]}
+          data={mappedFeatureImportance}
+        />
+      </Chart>
+    </div>
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
new file mode 100644
index 0000000000000..ebed1f25a3831
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+import { EuiCodeBlock } from '@elastic/eui';
+import { FeatureImportance } from '../../../../../common/types/feature_importance';
+
+interface DecisionPathJSONViewerProps {
+  featureImportance: FeatureImportance[];
+}
+export const DecisionPathJSONViewer: FC<DecisionPathJSONViewerProps> = ({ featureImportance }) => {
+  return <EuiCodeBlock>{JSON.stringify(featureImportance)}</EuiCodeBlock>;
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
new file mode 100644
index 0000000000000..2de3d6aa4e758
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC, useState } from 'react';
+import { EuiTabs, EuiTab } from '@elastic/eui';
+import { FeatureImportanceDecisionPath } from './decision_path_chart';
+import { DecisionPathJSONViewer } from './decision_path_json_viewer';
+import { FeatureImportance } from '../../../../../common/types/feature_importance';
+
+interface DecisionPathPopoverProps {
+  baseline?: number;
+  featureImportance: FeatureImportance[];
+}
+
+enum DECISION_PATH_TABS {
+  CHART = 'decision_path_chart',
+  JSON = 'decision_path_json',
+}
+export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
+  baseline,
+  featureImportance,
+}) => {
+  const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
+
+  if (featureImportance.length < 2) {
+    return <DecisionPathJSONViewer featureImportance={featureImportance} />;
+  }
+
+  const tabs = [
+    {
+      id: DECISION_PATH_TABS.CHART,
+      name: 'Chart',
+    },
+    {
+      id: DECISION_PATH_TABS.JSON,
+      name: 'JSON',
+    },
+  ];
+
+  return (
+    <div style={{ width: 300, height: 200 }}>
+      <EuiTabs display="condensed">
+        {tabs.map((tab) => (
+          <EuiTab
+            isSelected={tab.id === selectedTabId}
+            onClick={() => setSelectedTabId(tab.id)}
+            key={tab.id}
+          >
+            {tab.name}
+          </EuiTab>
+        ))}
+      </EuiTabs>
+      {selectedTabId === DECISION_PATH_TABS.CHART && (
+        <FeatureImportanceDecisionPath baseline={baseline} featureImportance={featureImportance} />
+      )}
+      {selectedTabId === DECISION_PATH_TABS.JSON && (
+        <DecisionPathJSONViewer featureImportance={featureImportance} />
+      )}
+    </div>
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
index 8395a11bd6fda..b27de033a9db6 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
@@ -181,6 +181,7 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
             </EuiFlexItem>
             <EuiFlexItem grow={false}>
               <DataGrid
+                jobConfig={jobConfig}
                 {...classificationData}
                 dataTestSubj="mlExplorationDataGrid"
                 toastNotifications={getToastNotifications()}
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 98dd40986e32b..9a93e35733a9a 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -52,7 +52,6 @@ export const useExplorationResults = (
       )
     );
   }
-
   const dataGrid = useDataGrid(
     columns,
     25,
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
index 7de39d91047ef..ffd1863f08449 100644
--- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
@@ -135,4 +135,12 @@ export const dataFrameAnalytics = {
       method: 'GET',
     });
   },
+  getAnalyticsBaseline(analyticsId, destinationIndex, predictionField) {
+    const body = JSON.stringify({ destinationIndex, predictionField });
+    return http<any>({
+      path: `${basePath()}/data_frame/analytics/baseline`,
+      method: 'POST',
+      body,
+    });
+  },
 };
diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
index 94feb21a6b5fb..bb50aa2c7f817 100644
--- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
@@ -16,6 +16,7 @@ import {
   analyticsIdSchema,
   stopsDataFrameAnalyticsJobQuerySchema,
   deleteDataFrameAnalyticsJobSchema,
+  getDataFrameAnalyticsBaselineSchema,
 } from './schemas/data_analytics_schema';
 import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns';
 import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics';
@@ -562,4 +563,64 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
       }
     })
   );
+
+  /**
+   * @apiGroup DataFrameAnalytics
+   *
+   * @api {get} /api/ml/data_frame/analytics/baseline Get analytics's feature importance baseline
+   * @apiName GetDataFrameAnalyticsBaseline
+   * @apiDescription Returns the baseline for data frame analytics job.
+   *
+   * @apiSchema (params) getDataFrameAnalyticsBaselineSchema
+   */
+  router.post(
+    {
+      path: '/api/ml/data_frame/analytics/baseline',
+      validate: {
+        body: getDataFrameAnalyticsBaselineSchema,
+      },
+      options: {
+        tags: ['access:ml:canGetDataFrameAnalytics'],
+      },
+    },
+    mlLicense.fullLicenseAPIGuard(async (context, request, response) => {
+      try {
+        const { destinationIndex, predictionField } = request.body;
+        const params = {
+          index: destinationIndex,
+          size: 0,
+          body: {
+            query: {
+              bool: {
+                filter: [
+                  {
+                    term: {
+                      'ml.is_training': true,
+                    },
+                  },
+                ],
+              },
+            },
+            aggs: {
+              featureImportanceBaseline: {
+                avg: {
+                  field: `ml.${predictionField}`,
+                },
+              },
+            },
+          },
+        };
+        let baseline;
+        const aggregationResult = await context.ml!.mlClient.callAsCurrentUser('search', params);
+        if (aggregationResult) {
+          baseline = aggregationResult.aggregations.featureImportanceBaseline.value;
+        }
+        return response.ok({
+          body: { baseline },
+        });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    })
+  );
 }
diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
index 0c3e186c314cc..a92586ecab9b3 100644
--- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
+++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
@@ -81,3 +81,11 @@ export const dataAnalyticsJobUpdateSchema = schema.object({
 export const stopsDataFrameAnalyticsJobQuerySchema = schema.object({
   force: schema.maybe(schema.boolean()),
 });
+
+export const getDataFrameAnalyticsBaselineSchema = schema.object({
+  /**
+   * Analytics Baseline
+   */
+  destinationIndex: schema.string(),
+  predictionField: schema.string(),
+});

From 031415ff9ca7a63323afa0e2ca54ae04084f782f Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 28 Jul 2020 15:50:30 -0500
Subject: [PATCH 02/37] [ML] Remove baseline div

---
 .../data_grid/feature_importance/decision_path_chart.tsx | 9 ---------
 .../feature_importance/decision_path_popover.tsx         | 2 +-
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 7347cad554352..535441a0d34b2 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -18,7 +18,6 @@ import {
   LineAnnotationDatum,
 } from '@elastic/charts';
 import _ from 'lodash';
-import { i18n } from '@kbn/i18n';
 import { EuiIcon } from '@elastic/eui';
 
 const style = {
@@ -82,14 +81,6 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
 
   return (
     <div style={{ width: 300, height: mappedFeatureImportance.length * 75 }}>
-      {baseline && (
-        <div>
-          {i18n.translate('xpack.ml.dataframe.analytics.explorationResults.decisionPathBaseline', {
-            defaultMessage: 'Baseline: {baseline}',
-            values: { baseline: baseline?.toFixed(3) },
-          })}
-        </div>
-      )}
       <Chart className="story-chart">
         <Settings rotation={90} />
         {baseline && (
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 2de3d6aa4e758..4170b446fbc79 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -42,7 +42,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
 
   return (
     <div style={{ width: 300, height: 200 }}>
-      <EuiTabs display="condensed">
+      <EuiTabs>
         {tabs.map((tab) => (
           <EuiTab
             isSelected={tab.id === selectedTabId}

From 40ef5e6a22fb2ed738c220d2d14bb3afede78ae5 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 28 Jul 2020 16:05:58 -0500
Subject: [PATCH 03/37] [ML] Move baseline logic to exploration results table

---
 .../components/data_grid/data_grid.tsx        | 38 ++-----------------
 .../exploration_results_table.tsx             | 12 ++++--
 .../use_exploration_results.ts                | 34 +++++++++++++++--
 3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 931c850d3357f..a26b8eb5580fd 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -5,7 +5,7 @@
  */
 
 import { isEqual } from 'lodash';
-import React, { memo, useEffect, FC, useState, useCallback } from 'react';
+import React, { memo, useEffect, FC } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import {
@@ -23,8 +23,6 @@ import {
 } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
-import { useMlKibana } from '../../contexts/kibana/kibana_context';
-
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
 
 import { DataFrameAnalyticsConfig, INDEX_STATUS } from '../../data_frame_analytics/common';
@@ -63,7 +61,8 @@ type Props = PropsWithHeader | PropsWithoutHeader;
 export const DataGrid: FC<Props> = memo(
   (props) => {
     const {
-      jobConfig,
+      baseline,
+      analyticsId,
       chartsVisible,
       chartsButtonVisible,
       columnsWithCharts,
@@ -85,12 +84,6 @@ export const DataGrid: FC<Props> = memo(
       toggleChartVisibility,
       visibleColumns,
     } = props;
-    const {
-      services: {
-        mlServices: { mlApiServices },
-      },
-    } = useMlKibana();
-
     // TODO Fix row hovering + bar highlighting
     // const getRowProps = (item: any) => {
     //   return {
@@ -99,29 +92,6 @@ export const DataGrid: FC<Props> = memo(
     //   };
     // };
 
-    const [baseline, setBaseLine] = useState();
-
-    const getAnalyticsBaseline = useCallback(async () => {
-      try {
-        const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
-          jobConfig.id,
-          jobConfig.dest.index,
-          jobConfig.analysis.classification?.prediction_field_name ??
-            jobConfig.analysis.regression?.prediction_field_name
-        );
-        if (result?.baseline) {
-          setBaseLine(result.baseline);
-        }
-      } catch (e) {
-        // eslint-disable-next-line
-        console.error(e);
-      }
-    }, [mlApiServices, jobConfig]);
-
-    useEffect(() => {
-      getAnalyticsBaseline();
-    }, [jobConfig]);
-
     useEffect(() => {
       if (invalidSortingColumnns.length > 0) {
         invalidSortingColumnns.forEach((columnId) => {
@@ -263,7 +233,7 @@ export const DataGrid: FC<Props> = memo(
                 const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
                 return (
                   <DecisionPathPopover
-                    analyticsId={jobConfig.id}
+                    analyticsId={analyticsId}
                     baseline={baseline}
                     featureImportance={parsedFIArray}
                   />
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
index b27de033a9db6..1a1b1ac508951 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
@@ -36,6 +36,7 @@ import { ExplorationQueryBar } from '../exploration_query_bar';
 import { IndexPatternPrompt } from '../index_pattern_prompt';
 
 import { useExplorationResults } from './use_exploration_results';
+import { useMlKibana } from '../../../../../contexts/kibana';
 
 const showingDocs = i18n.translate(
   'xpack.ml.dataframe.analytics.explorationResults.documentsShownHelpText',
@@ -70,6 +71,11 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
     setEvaluateSearchQuery,
     title,
   }) => {
+    const {
+      services: {
+        mlServices: { mlApiServices },
+      },
+    } = useMlKibana();
     const [searchQuery, setSearchQuery] = useState<SavedSearchQuery>(defaultSearchQuery);
 
     useEffect(() => {
@@ -80,8 +86,10 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
       indexPattern,
       jobConfig,
       searchQuery,
-      getToastNotifications()
+      getToastNotifications(),
+      mlApiServices
     );
+
     const docFieldsCount = classificationData.columnsWithCharts.length;
     const {
       columnsWithCharts,
@@ -94,7 +102,6 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
     if (jobConfig === undefined || classificationData === undefined) {
       return null;
     }
-
     // if it's a searchBar syntax error leave the table visible so they can try again
     if (status === INDEX_STATUS.ERROR && !errorMessage.includes('failed to create query')) {
       return (
@@ -181,7 +188,6 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
             </EuiFlexItem>
             <EuiFlexItem grow={false}>
               <DataGrid
-                jobConfig={jobConfig}
                 {...classificationData}
                 dataTestSubj="mlExplorationDataGrid"
                 toastNotifications={getToastNotifications()}
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 9a93e35733a9a..5957e80af2c9c 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -4,12 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useEffect, useMemo } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 
 import { EuiDataGridColumn } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
 
+import { MlApiServices } from '../../../../../services/ml_api_service';
 import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
 
 import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader';
@@ -23,7 +24,6 @@ import {
   UseIndexDataReturnType,
 } from '../../../../../components/data_grid';
 import { SavedSearchQuery } from '../../../../../contexts/ml';
-
 import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common';
 import {
   DEFAULT_RESULTS_FIELD,
@@ -36,8 +36,11 @@ export const useExplorationResults = (
   indexPattern: IndexPattern | undefined,
   jobConfig: DataFrameAnalyticsConfig | undefined,
   searchQuery: SavedSearchQuery,
-  toastNotifications: CoreSetup['notifications']['toasts']
+  toastNotifications: CoreSetup['notifications']['toasts'],
+  mlApiServices: MlApiServices
 ): UseIndexDataReturnType => {
+  const [baseline, setBaseLine] = useState();
+
   const needsDestIndexFields =
     indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0];
 
@@ -109,6 +112,29 @@ export const useExplorationResults = (
     JSON.stringify([searchQuery, dataGrid.visibleColumns]),
   ]);
 
+  const getAnalyticsBaseline = useCallback(async () => {
+    try {
+      if (jobConfig) {
+        const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
+          jobConfig.id,
+          jobConfig.dest.index,
+          jobConfig.analysis.classification?.prediction_field_name ??
+            jobConfig.analysis.regression?.prediction_field_name
+        );
+        if (result?.baseline) {
+          setBaseLine(result.baseline);
+        }
+      }
+    } catch (e) {
+      // eslint-disable-next-line
+      console.error(e);
+    }
+  }, [mlApiServices, jobConfig]);
+
+  useEffect(() => {
+    getAnalyticsBaseline();
+  }, [jobConfig]);
+
   const renderCellValue = useRenderCellValue(
     indexPattern,
     dataGrid.pagination,
@@ -119,5 +145,7 @@ export const useExplorationResults = (
   return {
     ...dataGrid,
     renderCellValue,
+    baseline,
+    analyticsId: jobConfig.id,
   };
 };

From 7e47db974356347fc96de189d9e25dcc0f83bef1 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 28 Jul 2020 16:55:07 -0500
Subject: [PATCH 04/37] [ML] Fix type issues

---
 .../application/components/data_grid/common.ts      |  2 +-
 .../application/components/data_grid/data_grid.tsx  | 11 +++--------
 .../feature_importance/decision_path_chart.tsx      |  4 ++--
 .../application/components/data_grid/types.ts       |  2 ++
 .../use_exploration_results.ts                      | 13 ++++++++-----
 .../services/ml_api_service/data_frame_analytics.ts |  2 +-
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
index 434a443c003f9..f252729cc20cd 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
@@ -5,7 +5,7 @@
  */
 
 import moment from 'moment-timezone';
-import React, { useEffect, useMemo } from 'react';
+import { useEffect, useMemo } from 'react';
 
 import {
   EuiDataGridCellValueElementProps,
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index a26b8eb5580fd..0ed4ee2ae7cf3 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -25,7 +25,7 @@ import {
 import { CoreSetup } from 'src/core/public';
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
 
-import { DataFrameAnalyticsConfig, INDEX_STATUS } from '../../data_frame_analytics/common';
+import { INDEX_STATUS } from '../../data_frame_analytics/common';
 
 import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
 import { UseIndexDataReturnType } from './types';
@@ -41,7 +41,7 @@ export const DataGridTitle: FC<{ title: string }> = ({ title }) => (
 );
 
 interface PropsWithoutHeader extends UseIndexDataReturnType {
-  jobConfig?: DataFrameAnalyticsConfig;
+  baseline?: number;
   dataTestSubj: string;
   toastNotifications: CoreSetup['notifications']['toasts'];
 }
@@ -62,7 +62,6 @@ export const DataGrid: FC<Props> = memo(
   (props) => {
     const {
       baseline,
-      analyticsId,
       chartsVisible,
       chartsButtonVisible,
       columnsWithCharts,
@@ -232,11 +231,7 @@ export const DataGrid: FC<Props> = memo(
                 const stringContents = cellContentsElement.textContent;
                 const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
                 return (
-                  <DecisionPathPopover
-                    analyticsId={analyticsId}
-                    baseline={baseline}
-                    featureImportance={parsedFIArray}
-                  />
+                  <DecisionPathPopover baseline={baseline} featureImportance={parsedFIArray} />
                 );
               },
             }}
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 535441a0d34b2..2b04f5da9865e 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -76,8 +76,8 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
     ]);
   }
 
-  const maxDomain = _.maxBy(featureImportance, (d) => d[1]);
-  const minDomain = _.minBy(featureImportance, (d) => d[1]);
+  const maxDomain = _.maxBy(mappedFeatureImportance, (d) => d[1]);
+  const minDomain = _.minBy(mappedFeatureImportance, (d) => d[1]);
 
   return (
     <div style={{ width: 300, height: mappedFeatureImportance.length * 75 }}>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/types.ts b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
index 756f74c8f9302..d7ff1296679b3 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/types.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
@@ -74,6 +74,7 @@ export interface UseIndexDataReturnType
     | 'tableItems'
     | 'toggleChartVisibility'
     | 'visibleColumns'
+    | 'baseline'
   > {
   renderCellValue: RenderCellValue;
 }
@@ -105,4 +106,5 @@ export interface UseDataGridReturnType {
   tableItems: DataGridItem[];
   toggleChartVisibility: () => void;
   visibleColumns: ColumnId[];
+  baseline?: number;
 }
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 5957e80af2c9c..3116750b41a69 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -24,7 +24,12 @@ import {
   UseIndexDataReturnType,
 } from '../../../../../components/data_grid';
 import { SavedSearchQuery } from '../../../../../contexts/ml';
-import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common';
+import {
+  getIndexData,
+  getIndexFields,
+  DataFrameAnalyticsConfig,
+  getPredictionFieldName,
+} from '../../../../common';
 import {
   DEFAULT_RESULTS_FIELD,
   FEATURE_IMPORTANCE,
@@ -114,12 +119,11 @@ export const useExplorationResults = (
 
   const getAnalyticsBaseline = useCallback(async () => {
     try {
-      if (jobConfig) {
+      if (jobConfig !== undefined && jobConfig.analysis !== undefined) {
         const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
           jobConfig.id,
           jobConfig.dest.index,
-          jobConfig.analysis.classification?.prediction_field_name ??
-            jobConfig.analysis.regression?.prediction_field_name
+          getPredictionFieldName(jobConfig.analysis)
         );
         if (result?.baseline) {
           setBaseLine(result.baseline);
@@ -146,6 +150,5 @@ export const useExplorationResults = (
     ...dataGrid,
     renderCellValue,
     baseline,
-    analyticsId: jobConfig.id,
   };
 };
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
index ffd1863f08449..2199d54245aec 100644
--- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
@@ -135,7 +135,7 @@ export const dataFrameAnalytics = {
       method: 'GET',
     });
   },
-  getAnalyticsBaseline(analyticsId, destinationIndex, predictionField) {
+  getAnalyticsBaseline(analyticsId: string, destinationIndex: string, predictionField?: string) {
     const body = JSON.stringify({ destinationIndex, predictionField });
     return http<any>({
       path: `${basePath()}/data_frame/analytics/baseline`,

From e8d0b41f47430a5a3b70d4196db427ec90194754 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 29 Jul 2020 10:36:20 -0500
Subject: [PATCH 05/37] [ML] Improvements to data viz charts

- Add grid lines
- Adjust sorting for classification from biggest at the top to smallest at the bottom
- Change chart to using cumulative values
---
 .../decision_path_chart.tsx                   | 28 +++++++++++++------
 .../decision_path_popover.tsx                 |  8 +++---
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 2b04f5da9865e..c0041e1dc67f9 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -65,22 +65,30 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
       difference: Math.abs(d[FEATURE_IMPORTANCE] - baseline),
     }));
 
-    // sort so decision path goes from bottom to top
+    // sort so importance so it goes from bottom (baseline) to top
     mappedFeatureImportance = mappedFeatureImportance
       .sort((a, b) => b.difference - a.difference)
       .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+
+    // start at the baseline and end at predicted value
+    let cumulativeSum = 0;
+    for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
+      cumulativeSum += mappedFeatureImportance[i][1];
+      mappedFeatureImportance[i][2] = cumulativeSum;
+    }
   } else {
-    mappedFeatureImportance = mappedFeatureImportance.map((d) => [
-      d[FEATURE_NAME],
-      d[FEATURE_IMPORTANCE],
-    ]);
+    // sort so most positive importance on top -> most negative importance at bottom
+    mappedFeatureImportance = mappedFeatureImportance
+      .sort((a, b) => b[FEATURE_IMPORTANCE] - a[FEATURE_IMPORTANCE])
+      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
   }
 
   const maxDomain = _.maxBy(mappedFeatureImportance, (d) => d[1]);
   const minDomain = _.minBy(mappedFeatureImportance, (d) => d[1]);
-
+  // adjust the height so it's compact for items with more features
+  const heightMultiplier = mappedFeatureImportance.length > 3 ? 20 : 50;
   return (
-    <div style={{ width: 300, height: mappedFeatureImportance.length * 75 }}>
+    <div style={{ height: mappedFeatureImportance.length * heightMultiplier }}>
       <Chart className="story-chart">
         <Settings rotation={90} />
         {baseline && (
@@ -94,6 +102,8 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         )}
 
         <Axis
+          title={'Prediction'}
+          showGridLines={true}
           id="bottom"
           position={Position.Bottom}
           showOverlappingTicks
@@ -106,13 +116,13 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
               : undefined
           }
         />
-        <Axis id="left" position={Position.Left} />
+        <Axis showGridLines={true} id="left" position={Position.Left} />
         <LineSeries
           id="importance"
           xScaleType={ScaleType.Ordinal}
           yScaleType={ScaleType.Linear}
           xAccessor={0}
-          yAccessors={[1]}
+          yAccessors={baseline ? [2] : [1]} // if baseline exist then shows the decision path else show in order of descending importance
           data={mappedFeatureImportance}
         />
       </Chart>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 4170b446fbc79..f7e5a7d3262b7 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -32,7 +32,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   const tabs = [
     {
       id: DECISION_PATH_TABS.CHART,
-      name: 'Chart',
+      name: 'Decision Plot',
     },
     {
       id: DECISION_PATH_TABS.JSON,
@@ -41,8 +41,8 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   ];
 
   return (
-    <div style={{ width: 300, height: 200 }}>
-      <EuiTabs>
+    <>
+      <EuiTabs size={'s'} style={{ width: 250 }}>
         {tabs.map((tab) => (
           <EuiTab
             isSelected={tab.id === selectedTabId}
@@ -59,6 +59,6 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
       {selectedTabId === DECISION_PATH_TABS.JSON && (
         <DecisionPathJSONViewer featureImportance={featureImportance} />
       )}
-    </div>
+    </>
   );
 };

From 5ad6690b512ced1d172c9b4d38449309fc74d2af Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 29 Jul 2020 14:25:16 -0500
Subject: [PATCH 06/37] [ML] Add info text

---
 .../decision_path_chart.tsx                   |  3 +-
 .../decision_path_popover.tsx                 | 68 +++++++++++++++----
 2 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index c0041e1dc67f9..70a9041290917 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -102,6 +102,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         )}
 
         <Axis
+          tickFormat={(d) => `${Number(d).toFixed(3)}`}
           title={'Prediction'}
           showGridLines={true}
           id="bottom"
@@ -118,7 +119,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         />
         <Axis showGridLines={true} id="left" position={Position.Left} />
         <LineSeries
-          id="importance"
+          id="prediction"
           xScaleType={ScaleType.Ordinal}
           yScaleType={ScaleType.Linear}
           xAccessor={0}
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index f7e5a7d3262b7..e866408f046d2 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -5,7 +5,8 @@
  */
 
 import React, { FC, useState } from 'react';
-import { EuiTabs, EuiTab } from '@elastic/eui';
+import { EuiTabs, EuiTab, EuiText, EuiLink } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
 import { FeatureImportanceDecisionPath } from './decision_path_chart';
 import { DecisionPathJSONViewer } from './decision_path_json_viewer';
 import { FeatureImportance } from '../../../../../common/types/feature_importance';
@@ -32,29 +33,66 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   const tabs = [
     {
       id: DECISION_PATH_TABS.CHART,
-      name: 'Decision Plot',
+      name: (
+        <FormattedMessage
+          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlot"
+          defaultMessage="Decision Plot"
+        />
+      ),
     },
     {
       id: DECISION_PATH_TABS.JSON,
-      name: 'JSON',
+      name: (
+        <FormattedMessage
+          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathJSON"
+          defaultMessage="JSON"
+        />
+      ),
     },
   ];
 
   return (
     <>
-      <EuiTabs size={'s'} style={{ width: 250 }}>
-        {tabs.map((tab) => (
-          <EuiTab
-            isSelected={tab.id === selectedTabId}
-            onClick={() => setSelectedTabId(tab.id)}
-            key={tab.id}
-          >
-            {tab.name}
-          </EuiTab>
-        ))}
-      </EuiTabs>
+      <div style={{ display: 'flex', width: 300 }}>
+        <EuiTabs size={'s'}>
+          {tabs.map((tab) => (
+            <EuiTab
+              isSelected={tab.id === selectedTabId}
+              onClick={() => setSelectedTabId(tab.id)}
+              key={tab.id}
+            >
+              {tab.name}
+            </EuiTab>
+          ))}
+        </EuiTabs>
+      </div>
       {selectedTabId === DECISION_PATH_TABS.CHART && (
-        <FeatureImportanceDecisionPath baseline={baseline} featureImportance={featureImportance} />
+        <>
+          <EuiText size={'xs'}>
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotHelpText"
+              defaultMessage={`SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted values.`}
+              values={{
+                linkedFeatureImportanceValues: (
+                  <EuiLink
+                    href="https://www.elastic.co/guide/en/machine-learning/7.8/ml-feature-importance.html"
+                    target="_blank"
+                  >
+                    <FormattedMessage
+                      id="xpack.monitoring.cluster.listing.incompatibleLicense.getLicenseLinkLabel"
+                      defaultMessage="feature importance values"
+                    />
+                  </EuiLink>
+                ),
+              }}
+            />
+          </EuiText>
+
+          <FeatureImportanceDecisionPath
+            baseline={baseline}
+            featureImportance={featureImportance}
+          />
+        </>
       )}
       {selectedTabId === DECISION_PATH_TABS.JSON && (
         <DecisionPathJSONViewer featureImportance={featureImportance} />

From ea12a63979adf7e5deb4a5d626c43d0430539ac0 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 29 Jul 2020 15:00:07 -0500
Subject: [PATCH 07/37] [ML] Change size to pass as Chart props

---
 .../feature_importance/decision_path_chart.tsx        | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 70a9041290917..d39fed9c6ca30 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -86,10 +86,13 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
   const maxDomain = _.maxBy(mappedFeatureImportance, (d) => d[1]);
   const minDomain = _.minBy(mappedFeatureImportance, (d) => d[1]);
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = mappedFeatureImportance.length > 3 ? 20 : 50;
+  const heightMultiplier = mappedFeatureImportance.length > 3 ? 20 : 75;
   return (
-    <div style={{ height: mappedFeatureImportance.length * heightMultiplier }}>
-      <Chart className="story-chart">
+    <>
+      <Chart
+        className="story-chart"
+        size={{ height: mappedFeatureImportance.length * heightMultiplier }}
+      >
         <Settings rotation={90} />
         {baseline && (
           <LineAnnotation
@@ -127,6 +130,6 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
           data={mappedFeatureImportance}
         />
       </Chart>
-    </div>
+    </>
   );
 };

From 49a541c41dcad267e4bcc8408bc8af3469a9d145 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 29 Jul 2020 15:15:37 -0500
Subject: [PATCH 08/37] [ML] Change to 3 sigfig instead of decimals

---
 .../data_grid/feature_importance/decision_path_chart.tsx        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index d39fed9c6ca30..d5adfff0129eb 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -105,7 +105,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         )}
 
         <Axis
-          tickFormat={(d) => `${Number(d).toFixed(3)}`}
+          tickFormat={(d) => `${Number(d).toPrecision(3)}`}
           title={'Prediction'}
           showGridLines={true}
           id="bottom"

From 52964e58172a553f03d6e92654717ed9c1d529d6 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 30 Jul 2020 09:10:12 -0500
Subject: [PATCH 09/37] [ML] Fix i18n issue

---
 .../data_grid/feature_importance/decision_path_popover.tsx    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index e866408f046d2..61029d412efaf 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -71,7 +71,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
           <EuiText size={'xs'}>
             <FormattedMessage
               id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotHelpText"
-              defaultMessage={`SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted values.`}
+              defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted values."
               values={{
                 linkedFeatureImportanceValues: (
                   <EuiLink
@@ -79,7 +79,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
                     target="_blank"
                   >
                     <FormattedMessage
-                      id="xpack.monitoring.cluster.listing.incompatibleLicense.getLicenseLinkLabel"
+                      id="xpack.ml.dataframe.analytics.explorationResults.linkedFeatureImportanceValues"
                       defaultMessage="feature importance values"
                     />
                   </EuiLink>

From 8b1e8c8ca7ded2fb8f8ced477264e153bf3251e0 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Mon, 10 Aug 2020 18:20:23 -0500
Subject: [PATCH 10/37] [ML] Decision path popover improvement + importance
 summary

---
 .../decision_path_chart.tsx                   |  79 ++++-------
 .../decision_path_popover.tsx                 |  53 +++++++-
 .../classification_exploration.tsx            |   3 +-
 .../feature_importance_summary.tsx            | 125 ++++++++++++++++++
 .../index.ts                                  |   7 +
 .../exploration_page_wrapper.tsx              |  19 ++-
 .../regression_exploration.tsx                |   2 +
 7 files changed, 228 insertions(+), 60 deletions(-)
 create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
 create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index d5adfff0129eb..ed7c6fd2b3520 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -7,7 +7,6 @@
 import React, { FC } from 'react';
 import {
   Chart,
-  Datum,
   Settings,
   LineAnnotation,
   AnnotationDomainTypes,
@@ -16,14 +15,15 @@ import {
   ScaleType,
   Position,
   LineAnnotationDatum,
+  PartialTheme,
 } from '@elastic/charts';
 import _ from 'lodash';
 import { EuiIcon } from '@elastic/eui';
 
-const style = {
+const baselineStyle = {
   line: {
     strokeWidth: 1,
-    stroke: '#ff0000',
+    stroke: 'gray',
     opacity: 1,
   },
   details: {
@@ -35,71 +35,42 @@ const style = {
   },
 };
 
-interface FeatureImportance {
-  feature_name: string;
-  importance: number;
-}
+const theme: PartialTheme = {
+  axes: {
+    tickLabel: {
+      fontSize: 30,
+    },
+  },
+};
+
 interface FeatureImportanceDecisionPathProps {
   baseline?: number;
-  featureImportance: FeatureImportance[];
+  decisionPlotData: LineAnnotationDatum[];
 }
 
-const FEATURE_NAME = 'feature_name';
-const FEATURE_IMPORTANCE = 'importance';
-
 export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProps> = ({
   baseline,
-  featureImportance,
+  decisionPlotData,
 }) => {
-  const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
-
-  let mappedFeatureImportance: Datum[] = featureImportance;
-
-  if (baseline) {
-    // also plot the baseline in the decision path visualization
-    mappedFeatureImportance.push({ [FEATURE_NAME]: 'baseline', [FEATURE_IMPORTANCE]: baseline });
-
-    // get the absolute difference of the importance value to the baseline for sorting
-    mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
-      ...d,
-      difference: Math.abs(d[FEATURE_IMPORTANCE] - baseline),
-    }));
-
-    // sort so importance so it goes from bottom (baseline) to top
-    mappedFeatureImportance = mappedFeatureImportance
-      .sort((a, b) => b.difference - a.difference)
-      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
-
-    // start at the baseline and end at predicted value
-    let cumulativeSum = 0;
-    for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
-      cumulativeSum += mappedFeatureImportance[i][1];
-      mappedFeatureImportance[i][2] = cumulativeSum;
-    }
-  } else {
-    // sort so most positive importance on top -> most negative importance at bottom
-    mappedFeatureImportance = mappedFeatureImportance
-      .sort((a, b) => b[FEATURE_IMPORTANCE] - a[FEATURE_IMPORTANCE])
-      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+  if (!decisionPlotData) {
+    return null;
   }
+  const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
+  const maxDomain = _.maxBy(decisionPlotData, (d) => d[2])[2];
+  const minDomain = _.minBy(decisionPlotData, (d) => d[2])[2];
 
-  const maxDomain = _.maxBy(mappedFeatureImportance, (d) => d[1]);
-  const minDomain = _.minBy(mappedFeatureImportance, (d) => d[1]);
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = mappedFeatureImportance.length > 3 ? 20 : 75;
+  const heightMultiplier = decisionPlotData.length > 3 ? 20 : 75;
   return (
     <>
-      <Chart
-        className="story-chart"
-        size={{ height: mappedFeatureImportance.length * heightMultiplier }}
-      >
-        <Settings rotation={90} />
+      <Chart className="story-chart" size={{ height: decisionPlotData.length * heightMultiplier }}>
+        <Settings theme={theme} rotation={90} />
         {baseline && (
           <LineAnnotation
             id="dfa_baseline"
             domainType={AnnotationDomainTypes.YDomain}
             dataValues={baselineData}
-            style={style}
+            style={baselineStyle}
             marker={<EuiIcon type={'annotation'} />}
           />
         )}
@@ -114,8 +85,8 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
           domain={
             minDomain && maxDomain
               ? {
-                  min: minDomain.importance,
-                  max: maxDomain.importance,
+                  min: minDomain,
+                  max: maxDomain,
                 }
               : undefined
           }
@@ -127,7 +98,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
           yScaleType={ScaleType.Linear}
           xAccessor={0}
           yAccessors={baseline ? [2] : [1]} // if baseline exist then shows the decision path else show in order of descending importance
-          data={mappedFeatureImportance}
+          data={decisionPlotData}
         />
       </Chart>
     </>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 61029d412efaf..d4568a8f2b88f 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { FC, useState } from 'react';
+import React, { FC, useEffect, useState } from 'react';
 import { EuiTabs, EuiTab, EuiText, EuiLink } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { FeatureImportanceDecisionPath } from './decision_path_chart';
@@ -20,11 +20,57 @@ enum DECISION_PATH_TABS {
   CHART = 'decision_path_chart',
   JSON = 'decision_path_json',
 }
+
+const FEATURE_NAME = 'feature_name';
+const FEATURE_IMPORTANCE = 'importance';
+
+export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPathPopoverProps) => {
+  const [decisionPlotData, setDecisionPlotData] = useState();
+
+  useEffect(() => {
+    let mappedFeatureImportance: FeatureImportance[] = featureImportance;
+
+    if (baseline) {
+      // get the absolute difference of the importance value to the baseline for sorting
+
+      mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
+        ...d,
+        difference: Math.abs(d[FEATURE_IMPORTANCE]),
+      }));
+      mappedFeatureImportance.push({
+        [FEATURE_NAME]: 'baseline',
+        [FEATURE_IMPORTANCE]: baseline,
+        difference: 0,
+      });
+
+      // sort so importance so it goes from bottom (baseline) to top
+      mappedFeatureImportance = mappedFeatureImportance
+        .sort((a, b) => b.difference - a.difference)
+        .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+
+      // start at the baseline and end at predicted value
+      let cumulativeSum = 0;
+      for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
+        cumulativeSum += mappedFeatureImportance[i][1];
+        mappedFeatureImportance[i][2] = cumulativeSum;
+      }
+    } else {
+      // sort so most positive importance on top -> most negative importance at bottom
+      mappedFeatureImportance = featureImportance
+        // .sort((a, b) => b[FEATURE_IMPORTANCE] - a[FEATURE_IMPORTANCE])
+        .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+    }
+    setDecisionPlotData(mappedFeatureImportance);
+  }, [baseline, featureImportance]);
+
+  return { decisionPlotData };
+};
 export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   baseline,
   featureImportance,
 }) => {
   const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
+  const { decisionPlotData } = useDecisionPathData({ baseline, featureImportance });
 
   if (featureImportance.length < 2) {
     return <DecisionPathJSONViewer featureImportance={featureImportance} />;
@@ -36,7 +82,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
       name: (
         <FormattedMessage
           id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlot"
-          defaultMessage="Decision Plot"
+          defaultMessage="Decision plot"
         />
       ),
     },
@@ -68,7 +114,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
       </div>
       {selectedTabId === DECISION_PATH_TABS.CHART && (
         <>
-          <EuiText size={'xs'}>
+          <EuiText size={'xs'} color="subdued" style={{ paddingTop: 5 }}>
             <FormattedMessage
               id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotHelpText"
               defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted values."
@@ -91,6 +137,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
           <FeatureImportanceDecisionPath
             baseline={baseline}
             featureImportance={featureImportance}
+            decisionPlotData={decisionPlotData}
           />
         </>
       )}
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
index ccac9a697210b..4763dc65bc9d3 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
@@ -9,8 +9,8 @@ import React, { FC } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import { ExplorationPageWrapper } from '../exploration_page_wrapper';
-
 import { EvaluatePanel } from './evaluate_panel';
+import { FeatureImportanceSummary } from '../exploration_feature_importance_summary';
 
 interface Props {
   jobId: string;
@@ -28,6 +28,7 @@ export const ClassificationExploration: FC<Props> = ({ jobId }) => {
         }
       )}
       EvaluatePanel={EvaluatePanel}
+      FeatureImportanceSummary={FeatureImportanceSummary}
     />
   );
 };
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
new file mode 100644
index 0000000000000..07be7df162b99
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+  EuiButtonEmpty,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiIconTip,
+  EuiPanel,
+  EuiSpacer,
+  EuiTitle,
+} from '@elastic/eui';
+import { Chart, Settings, Axis, ScaleType, Position, BarSeries } from '@elastic/charts';
+import { useMlKibana } from '../../../../../contexts/kibana';
+
+const mockData = [
+  {
+    feature_name: 'g4',
+    importance: {
+      mean: 233.4600671221131,
+      variance: 16372.967040744528,
+      min: -565.7664157184156,
+      max: 468.8953979253651,
+    },
+  },
+  {
+    feature_name: 'g3',
+    importance: {
+      mean: 227.52681995349807,
+      variance: 15305.348788185232,
+      min: -474.187447119175,
+      max: 500.79764582176218,
+    },
+  },
+  {
+    feature_name: 'g1',
+    importance: {
+      mean: 479.8491325534919,
+      variance: 1056.2609531379472,
+      min: -584.6059620924166,
+      max: 601.3424189083114,
+    },
+  },
+  {
+    feature_name: 'g2',
+    importance: {
+      mean: 729.7375145579323,
+      variance: 173166.9595517114,
+      min: -1438.469059491588,
+      max: 1428.738023747545,
+    },
+  },
+]
+  .sort((a, b) => b.importance.mean - a.importance.mean)
+  .map((d) => [d.feature_name, d.importance.mean]);
+
+const tooltipContent = i18n.translate(
+  'xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTooltipContent',
+  {
+    defaultMessage:
+      'Shows to what degree a given feature of a data point contributes to the prediction. The magnitude of feature importance shows how significantly the feature affects the prediction both locally (for a given data point) or generally (for the whole data set).',
+  }
+);
+export const FeatureImportanceSummary = () => {
+  const {
+    services: { docLinks },
+  } = useMlKibana();
+
+  const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks;
+
+  return (
+    <EuiPanel>
+      <EuiFlexGroup>
+        <EuiFlexItem grow={false}>
+          <EuiTitle size="xs">
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTitle"
+              defaultMessage="Feature importance summary"
+            />
+          </EuiTitle>
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <EuiIconTip content={tooltipContent} />
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiSpacer />
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <EuiButtonEmpty
+            target="_blank"
+            iconType="help"
+            iconSide="left"
+            color="primary"
+            href={`${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-feature-importance.html`}
+          >
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.exploration.featureImportanceDocsLink"
+              defaultMessage="Feature importance docs"
+            />
+          </EuiButtonEmpty>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+      <Chart className="story-chart" size={{ width: '100%', height: 300 }}>
+        <Settings rotation={90} />
+
+        <Axis id="x-axis" title="Feature importance average magnitude" position={Position.Bottom} />
+        <Axis id="y-axis" title="" position={Position.Left} />
+        <BarSeries
+          id="bars"
+          xScaleType={ScaleType.Ordinal}
+          yScaleType={ScaleType.Linear}
+          xAccessor={0}
+          yAccessors={[1]}
+          data={mockData}
+        />
+      </Chart>
+    </EuiPanel>
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts
new file mode 100644
index 0000000000000..5c447b40feb77
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { FeatureImportanceSummary } from './feature_importance_summary';
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
index 34ff36c59fa6c..c97e03b8095a4 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
@@ -27,9 +27,15 @@ interface Props {
   jobId: string;
   title: string;
   EvaluatePanel: FC<EvaluatePanelProps>;
+  FeatureImportanceSummaryPanel?: FC<EvaluatePanelProps>;
 }
 
-export const ExplorationPageWrapper: FC<Props> = ({ jobId, title, EvaluatePanel }) => {
+export const ExplorationPageWrapper: FC<Props> = ({
+  jobId,
+  title,
+  EvaluatePanel,
+  FeatureImportanceSummaryPanel,
+}) => {
   const {
     indexPattern,
     isInitialized,
@@ -51,7 +57,6 @@ export const ExplorationPageWrapper: FC<Props> = ({ jobId, title, EvaluatePanel
       />
     );
   }
-
   return (
     <>
       {isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
@@ -59,6 +64,16 @@ export const ExplorationPageWrapper: FC<Props> = ({ jobId, title, EvaluatePanel
         <EvaluatePanel jobConfig={jobConfig} jobStatus={jobStatus} searchQuery={searchQuery} />
       )}
       <EuiSpacer />
+      {FeatureImportanceSummaryPanel !== undefined && (
+        <>
+          <FeatureImportanceSummaryPanel
+            jobConfig={jobConfig}
+            jobStatus={jobStatus}
+            searchQuery={searchQuery}
+          />
+          <EuiSpacer />
+        </>
+      )}
       {isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
       {isLoadingJobConfig === false &&
         jobConfig !== undefined &&
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
index 36d91f6f41d44..c27b95b8e3aae 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
@@ -11,6 +11,7 @@ import { i18n } from '@kbn/i18n';
 import { ExplorationPageWrapper } from '../exploration_page_wrapper';
 
 import { EvaluatePanel } from './evaluate_panel';
+import { FeatureImportanceSummary } from '../exploration_feature_importance_summary';
 
 interface Props {
   jobId: string;
@@ -25,6 +26,7 @@ export const RegressionExploration: FC<Props> = ({ jobId }) => {
         values: { jobId },
       })}
       EvaluatePanel={EvaluatePanel}
+      FeatureImportanceSummaryPanel={FeatureImportanceSummary}
     />
   );
 };

From 19a1aae5231ca532eee1e68e9a4b135abbfa8f2f Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 18 Aug 2020 09:18:04 -0500
Subject: [PATCH 11/37] [ML] Change title to fit summary

---
 .../feature_importance_summary.tsx                     | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
index 07be7df162b99..a9e2e6206cf1d 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
@@ -79,10 +79,12 @@ export const FeatureImportanceSummary = () => {
       <EuiFlexGroup>
         <EuiFlexItem grow={false}>
           <EuiTitle size="xs">
-            <FormattedMessage
-              id="xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTitle"
-              defaultMessage="Feature importance summary"
-            />
+            <span>
+              <FormattedMessage
+                id="xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTitle"
+                defaultMessage="Feature importance summary"
+              />
+            </span>
           </EuiTitle>
         </EuiFlexItem>
         <EuiFlexItem grow={false}>

From c718437496ae78e58d05c01b9e817faf6e336865 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 18 Aug 2020 11:31:11 -0500
Subject: [PATCH 12/37] [ML] Consolidate feature importance

---
 .../decision_path_chart.tsx                   | 12 +++--
 .../decision_path_json_viewer.tsx             |  2 +-
 .../decision_path_popover.tsx                 | 50 ++++++++++---------
 .../ml/server/routes/data_frame_analytics.ts  |  4 +-
 4 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index ed7c6fd2b3520..e9bf0315a3b94 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -56,11 +56,13 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
     return null;
   }
   const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
-  const maxDomain = _.maxBy(decisionPlotData, (d) => d[2])[2];
-  const minDomain = _.minBy(decisionPlotData, (d) => d[2])[2];
-
+  let maxDomain = _.maxBy(decisionPlotData, (d) => d[2])[2];
+  let minDomain = _.minBy(decisionPlotData, (d) => d[2])[2];
+  const buffer = Math.abs(maxDomain - minDomain) * 0.1;
+  maxDomain = maxDomain + buffer;
+  minDomain = minDomain - buffer;
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = decisionPlotData.length > 3 ? 20 : 75;
+  const heightMultiplier = decisionPlotData.length > 3 ? 35 : 75;
   return (
     <>
       <Chart className="story-chart" size={{ height: decisionPlotData.length * heightMultiplier }}>
@@ -97,7 +99,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
           xScaleType={ScaleType.Ordinal}
           yScaleType={ScaleType.Linear}
           xAccessor={0}
-          yAccessors={baseline ? [2] : [1]} // if baseline exist then shows the decision path else show in order of descending importance
+          yAccessors={[2]} // if baseline exist then shows the decision path else show in order of descending importance
           data={decisionPlotData}
         />
       </Chart>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
index ebed1f25a3831..343324b27f9b5 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_json_viewer.tsx
@@ -12,5 +12,5 @@ interface DecisionPathJSONViewerProps {
   featureImportance: FeatureImportance[];
 }
 export const DecisionPathJSONViewer: FC<DecisionPathJSONViewerProps> = ({ featureImportance }) => {
-  return <EuiCodeBlock>{JSON.stringify(featureImportance)}</EuiCodeBlock>;
+  return <EuiCodeBlock isCopyable={true}>{JSON.stringify(featureImportance)}</EuiCodeBlock>;
 };
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index d4568a8f2b88f..c0e57849e96a8 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -24,42 +24,44 @@ enum DECISION_PATH_TABS {
 const FEATURE_NAME = 'feature_name';
 const FEATURE_IMPORTANCE = 'importance';
 
+export interface ExtendedFeatureImportance extends FeatureImportance {
+  absImportance?: number;
+}
 export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPathPopoverProps) => {
   const [decisionPlotData, setDecisionPlotData] = useState();
 
   useEffect(() => {
-    let mappedFeatureImportance: FeatureImportance[] = featureImportance;
+    let mappedFeatureImportance: ExtendedFeatureImportance[] = featureImportance;
+    mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
+      ...d,
+      absImportance: Math.abs(d[FEATURE_IMPORTANCE]),
+    }));
 
     if (baseline) {
-      // get the absolute difference of the importance value to the baseline for sorting
-
-      mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
-        ...d,
-        difference: Math.abs(d[FEATURE_IMPORTANCE]),
-      }));
+      // get the absolute absImportance of the importance value to the baseline for sorting
       mappedFeatureImportance.push({
         [FEATURE_NAME]: 'baseline',
         [FEATURE_IMPORTANCE]: baseline,
-        difference: 0,
+        absImportance: 0,
       });
+    }
 
-      // sort so importance so it goes from bottom (baseline) to top
-      mappedFeatureImportance = mappedFeatureImportance
-        .sort((a, b) => b.difference - a.difference)
-        .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
-
-      // start at the baseline and end at predicted value
-      let cumulativeSum = 0;
-      for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
-        cumulativeSum += mappedFeatureImportance[i][1];
-        mappedFeatureImportance[i][2] = cumulativeSum;
-      }
-    } else {
-      // sort so most positive importance on top -> most negative importance at bottom
-      mappedFeatureImportance = featureImportance
-        // .sort((a, b) => b[FEATURE_IMPORTANCE] - a[FEATURE_IMPORTANCE])
-        .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+    mappedFeatureImportance = mappedFeatureImportance
+      // sort so absolute importance so it goes from bottom (baseline) to top
+      .sort((a, b) => b.absImportance - a.absImportance)
+      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+
+    // start at the baseline and end at predicted value
+    // for regression, cumulativeSum should add up to baseline
+    let cumulativeSum = 0;
+    for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
+      cumulativeSum += mappedFeatureImportance[i][1];
+      mappedFeatureImportance[i][2] = cumulativeSum;
     }
+
+    // go back and readjust the starting point to m.{}_prediction - cumulativeSum
+    // to account for when there are more or less features included in the analysis than the max set
+
     setDecisionPlotData(mappedFeatureImportance);
   }, [baseline, featureImportance]);
 
diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
index 178d8d502bd96..f02df32234f5a 100644
--- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
@@ -560,7 +560,7 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
         tags: ['access:ml:canGetDataFrameAnalytics'],
       },
     },
-    mlLicense.fullLicenseAPIGuard(async (context, request, response) => {
+    mlLicense.fullLicenseAPIGuard(async ({ legacyClient, request, response }) => {
       try {
         const { destinationIndex, predictionField } = request.body;
         const params = {
@@ -588,7 +588,7 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
           },
         };
         let baseline;
-        const aggregationResult = await context.ml!.mlClient.callAsCurrentUser('search', params);
+        const aggregationResult = await legacyClient.callAsCurrentUser('search', params);
         if (aggregationResult) {
           baseline = aggregationResult.aggregations.featureImportanceBaseline.value;
         }

From d0fa2453feef51624d23f201c0fb7483cd6ac1b2 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 18 Aug 2020 15:14:35 -0500
Subject: [PATCH 13/37] [ML] useMemo for popoverContent

---
 .../components/data_grid/data_grid.tsx        | 22 ++++++++++---------
 .../decision_path_chart.tsx                   |  2 +-
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 0ed4ee2ae7cf3..f9560d7290835 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -5,7 +5,7 @@
  */
 
 import { isEqual } from 'lodash';
-import React, { memo, useEffect, FC } from 'react';
+import React, { memo, useEffect, FC, useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import {
@@ -91,6 +91,16 @@ export const DataGrid: FC<Props> = memo(
     //   };
     // };
 
+    const popOverContent = useMemo(() => {
+      return {
+        featureImportance: ({ cellContentsElement }) => {
+          const stringContents = cellContentsElement.textContent;
+          const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
+          return <DecisionPathPopover baseline={baseline} featureImportance={parsedFIArray} />;
+        },
+      };
+    }, [baseline]);
+
     useEffect(() => {
       if (invalidSortingColumnns.length > 0) {
         invalidSortingColumnns.forEach((columnId) => {
@@ -226,15 +236,7 @@ export const DataGrid: FC<Props> = memo(
                   }
                 : {}),
             }}
-            popoverContents={{
-              featureImportance: ({ cellContentsElement }) => {
-                const stringContents = cellContentsElement.textContent;
-                const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
-                return (
-                  <DecisionPathPopover baseline={baseline} featureImportance={parsedFIArray} />
-                );
-              },
-            }}
+            popoverContents={popOverContent}
             pagination={{
               ...pagination,
               pageSizeOptions: [5, 10, 25],
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index e9bf0315a3b94..ffa6f29504610 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -62,7 +62,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
   maxDomain = maxDomain + buffer;
   minDomain = minDomain - buffer;
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = decisionPlotData.length > 3 ? 35 : 75;
+  const heightMultiplier = decisionPlotData.length > 3 ? 25 : 75;
   return (
     <>
       <Chart className="story-chart" size={{ height: decisionPlotData.length * heightMultiplier }}>

From a317505f5f58d60bf46fa94af7899a9dedfa3924 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 18 Aug 2020 16:14:23 -0500
Subject: [PATCH 14/37] [ML] Add adjustment to baseline

---
 .../components/data_grid/data_grid.tsx        | 19 ++++++++++--
 .../decision_path_chart.tsx                   | 14 ++++++---
 .../decision_path_popover.tsx                 | 31 +++++++++++++------
 .../application/components/data_grid/types.ts |  2 ++
 .../use_exploration_results.ts                |  4 ++-
 5 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index f9560d7290835..ea02194df46c0 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -82,6 +82,7 @@ export const DataGrid: FC<Props> = memo(
       toastNotifications,
       toggleChartVisibility,
       visibleColumns,
+      predictionFieldName,
     } = props;
     // TODO Fix row hovering + bar highlighting
     // const getRowProps = (item: any) => {
@@ -93,13 +94,25 @@ export const DataGrid: FC<Props> = memo(
 
     const popOverContent = useMemo(() => {
       return {
-        featureImportance: ({ cellContentsElement }) => {
+        featureImportance: ({ cellContentsElement, children }) => {
           const stringContents = cellContentsElement.textContent;
           const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
-          return <DecisionPathPopover baseline={baseline} featureImportance={parsedFIArray} />;
+          const rowIndex = children?.props?.rowIndex;
+          const row = data[rowIndex];
+          let predictedValue;
+          if (row && predictionFieldName && row.ml[predictionFieldName] !== undefined) {
+            predictedValue = row.ml[predictionFieldName];
+          }
+          return (
+            <DecisionPathPopover
+              predictedValue={predictedValue}
+              baseline={baseline}
+              featureImportance={parsedFIArray}
+            />
+          );
         },
       };
-    }, [baseline]);
+    }, [baseline, data]);
 
     useEffect(() => {
       if (invalidSortingColumnns.length > 0) {
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index ffa6f29504610..91e3950a0ee04 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -46,23 +46,27 @@ const theme: PartialTheme = {
 interface FeatureImportanceDecisionPathProps {
   baseline?: number;
   decisionPlotData: LineAnnotationDatum[];
+  predictedValue?: number | undefined;
 }
 
 export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProps> = ({
   baseline,
   decisionPlotData,
 }) => {
-  if (!decisionPlotData) {
-    return null;
-  }
+  if (!decisionPlotData) return <div />;
+
   const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
   let maxDomain = _.maxBy(decisionPlotData, (d) => d[2])[2];
   let minDomain = _.minBy(decisionPlotData, (d) => d[2])[2];
+  // adjust domain so plot have some space on both sides
+  // and to account for baseline out of range
   const buffer = Math.abs(maxDomain - minDomain) * 0.1;
-  maxDomain = maxDomain + buffer;
-  minDomain = minDomain - buffer;
+  maxDomain = Math.max(maxDomain, baseline) + buffer;
+  minDomain = Math.min(minDomain, baseline) - buffer;
+
   // adjust the height so it's compact for items with more features
   const heightMultiplier = decisionPlotData.length > 3 ? 25 : 75;
+
   return (
     <>
       <Chart className="story-chart" size={{ height: decisionPlotData.length * heightMultiplier }}>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index c0e57849e96a8..f15a6ca33ebaf 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -14,6 +14,7 @@ import { FeatureImportance } from '../../../../../common/types/feature_importanc
 interface DecisionPathPopoverProps {
   baseline?: number;
   featureImportance: FeatureImportance[];
+  predictedValue?: number | undefined;
 }
 
 enum DECISION_PATH_TABS {
@@ -27,7 +28,11 @@ const FEATURE_IMPORTANCE = 'importance';
 export interface ExtendedFeatureImportance extends FeatureImportance {
   absImportance?: number;
 }
-export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPathPopoverProps) => {
+export const useDecisionPathData = ({
+  baseline,
+  featureImportance,
+  predictedValue,
+}: DecisionPathPopoverProps) => {
   const [decisionPlotData, setDecisionPlotData] = useState();
 
   useEffect(() => {
@@ -37,11 +42,21 @@ export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPat
       absImportance: Math.abs(d[FEATURE_IMPORTANCE]),
     }));
 
-    if (baseline) {
+    if (baseline && Number.isFinite(predictedValue)) {
+      // get the adjusted importance needed for when # of fields included in c++ analysis != max allowed
+      // if num fields included = num features allowed exactly, adjustedImportance should be 0
+      const adjustedImportance =
+        predictedValue -
+        mappedFeatureImportance.reduce(
+          (accumulator, currentValue) => accumulator + currentValue.importance,
+          0
+        ) -
+        baseline;
+
       // get the absolute absImportance of the importance value to the baseline for sorting
       mappedFeatureImportance.push({
-        [FEATURE_NAME]: 'baseline',
-        [FEATURE_IMPORTANCE]: baseline,
+        [FEATURE_NAME]: 'other',
+        [FEATURE_IMPORTANCE]: baseline + adjustedImportance,
         absImportance: 0,
       });
     }
@@ -58,10 +73,6 @@ export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPat
       cumulativeSum += mappedFeatureImportance[i][1];
       mappedFeatureImportance[i][2] = cumulativeSum;
     }
-
-    // go back and readjust the starting point to m.{}_prediction - cumulativeSum
-    // to account for when there are more or less features included in the analysis than the max set
-
     setDecisionPlotData(mappedFeatureImportance);
   }, [baseline, featureImportance]);
 
@@ -70,9 +81,10 @@ export const useDecisionPathData = ({ baseline, featureImportance }: DecisionPat
 export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   baseline,
   featureImportance,
+  predictedValue,
 }) => {
   const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
-  const { decisionPlotData } = useDecisionPathData({ baseline, featureImportance });
+  const { decisionPlotData } = useDecisionPathData({ baseline, featureImportance, predictedValue });
 
   if (featureImportance.length < 2) {
     return <DecisionPathJSONViewer featureImportance={featureImportance} />;
@@ -140,6 +152,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
             baseline={baseline}
             featureImportance={featureImportance}
             decisionPlotData={decisionPlotData}
+            predictedValue={predictedValue}
           />
         </>
       )}
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/types.ts b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
index d7ff1296679b3..c2cf6d28c2794 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/types.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
@@ -75,6 +75,7 @@ export interface UseIndexDataReturnType
     | 'toggleChartVisibility'
     | 'visibleColumns'
     | 'baseline'
+    | 'predictionFieldName'
   > {
   renderCellValue: RenderCellValue;
 }
@@ -107,4 +108,5 @@ export interface UseDataGridReturnType {
   toggleChartVisibility: () => void;
   visibleColumns: ColumnId[];
   baseline?: number;
+  predictionFieldName?: string;
 }
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 338559f6137a9..92edde5d347a5 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -114,6 +114,7 @@ export const useExplorationResults = (
     jobConfig?.dest.index,
     JSON.stringify([searchQuery, dataGrid.visibleColumns]),
   ]);
+  const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
 
   const getAnalyticsBaseline = useCallback(async () => {
     try {
@@ -121,7 +122,7 @@ export const useExplorationResults = (
         const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
           jobConfig.id,
           jobConfig.dest.index,
-          getPredictionFieldName(jobConfig.analysis)
+          predictionFieldName
         );
         if (result?.baseline) {
           setBaseLine(result.baseline);
@@ -148,5 +149,6 @@ export const useExplorationResults = (
     ...dataGrid,
     renderCellValue,
     baseline,
+    predictionFieldName,
   };
 };

From 973ddb92e61fde07b9bf6e7aebbae62aeab0e3d4 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 18 Aug 2020 17:00:27 -0500
Subject: [PATCH 15/37] [ML] Call /baseline only if it's a regression analysis
 for now

---
 .../exploration_results_table/use_exploration_results.ts   | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 92edde5d347a5..616dd6bba1503 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -36,6 +36,7 @@ import {
   TOP_CLASSES,
 } from '../../../../common/constants';
 import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields';
+import { isRegressionAnalysis } from '../../../../common/analytics';
 
 export const useExplorationResults = (
   indexPattern: IndexPattern | undefined,
@@ -118,7 +119,11 @@ export const useExplorationResults = (
 
   const getAnalyticsBaseline = useCallback(async () => {
     try {
-      if (jobConfig !== undefined && jobConfig.analysis !== undefined) {
+      if (
+        jobConfig !== undefined &&
+        jobConfig.analysis !== undefined &&
+        isRegressionAnalysis(jobConfig.analysis)
+      ) {
         const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
           jobConfig.id,
           jobConfig.dest.index,

From 39e374911acf0930fe5cab3966c361c2d02ed4d6 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 19 Aug 2020 11:08:52 -0500
Subject: [PATCH 16/37] [ML] Remove feature importance summary bars for now

---
 .../classification_exploration.tsx            |   2 -
 .../feature_importance_summary.tsx            | 127 ------------------
 .../index.ts                                  |   7 -
 .../exploration_page_wrapper.tsx              |  18 +--
 .../regression_exploration.tsx                |   2 -
 5 files changed, 1 insertion(+), 155 deletions(-)
 delete mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
 delete mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
index 4763dc65bc9d3..2e3a5d89367ce 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/classification_exploration/classification_exploration.tsx
@@ -10,7 +10,6 @@ import { i18n } from '@kbn/i18n';
 
 import { ExplorationPageWrapper } from '../exploration_page_wrapper';
 import { EvaluatePanel } from './evaluate_panel';
-import { FeatureImportanceSummary } from '../exploration_feature_importance_summary';
 
 interface Props {
   jobId: string;
@@ -28,7 +27,6 @@ export const ClassificationExploration: FC<Props> = ({ jobId }) => {
         }
       )}
       EvaluatePanel={EvaluatePanel}
-      FeatureImportanceSummary={FeatureImportanceSummary}
     />
   );
 };
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
deleted file mode 100644
index a9e2e6206cf1d..0000000000000
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/feature_importance_summary.tsx
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import React from 'react';
-import { i18n } from '@kbn/i18n';
-import { FormattedMessage } from '@kbn/i18n/react';
-import {
-  EuiButtonEmpty,
-  EuiFlexGroup,
-  EuiFlexItem,
-  EuiIconTip,
-  EuiPanel,
-  EuiSpacer,
-  EuiTitle,
-} from '@elastic/eui';
-import { Chart, Settings, Axis, ScaleType, Position, BarSeries } from '@elastic/charts';
-import { useMlKibana } from '../../../../../contexts/kibana';
-
-const mockData = [
-  {
-    feature_name: 'g4',
-    importance: {
-      mean: 233.4600671221131,
-      variance: 16372.967040744528,
-      min: -565.7664157184156,
-      max: 468.8953979253651,
-    },
-  },
-  {
-    feature_name: 'g3',
-    importance: {
-      mean: 227.52681995349807,
-      variance: 15305.348788185232,
-      min: -474.187447119175,
-      max: 500.79764582176218,
-    },
-  },
-  {
-    feature_name: 'g1',
-    importance: {
-      mean: 479.8491325534919,
-      variance: 1056.2609531379472,
-      min: -584.6059620924166,
-      max: 601.3424189083114,
-    },
-  },
-  {
-    feature_name: 'g2',
-    importance: {
-      mean: 729.7375145579323,
-      variance: 173166.9595517114,
-      min: -1438.469059491588,
-      max: 1428.738023747545,
-    },
-  },
-]
-  .sort((a, b) => b.importance.mean - a.importance.mean)
-  .map((d) => [d.feature_name, d.importance.mean]);
-
-const tooltipContent = i18n.translate(
-  'xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTooltipContent',
-  {
-    defaultMessage:
-      'Shows to what degree a given feature of a data point contributes to the prediction. The magnitude of feature importance shows how significantly the feature affects the prediction both locally (for a given data point) or generally (for the whole data set).',
-  }
-);
-export const FeatureImportanceSummary = () => {
-  const {
-    services: { docLinks },
-  } = useMlKibana();
-
-  const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks;
-
-  return (
-    <EuiPanel>
-      <EuiFlexGroup>
-        <EuiFlexItem grow={false}>
-          <EuiTitle size="xs">
-            <span>
-              <FormattedMessage
-                id="xpack.ml.dataframe.analytics.exploration.featureImportanceSummaryTitle"
-                defaultMessage="Feature importance summary"
-              />
-            </span>
-          </EuiTitle>
-        </EuiFlexItem>
-        <EuiFlexItem grow={false}>
-          <EuiIconTip content={tooltipContent} />
-        </EuiFlexItem>
-        <EuiFlexItem>
-          <EuiSpacer />
-        </EuiFlexItem>
-        <EuiFlexItem grow={false}>
-          <EuiButtonEmpty
-            target="_blank"
-            iconType="help"
-            iconSide="left"
-            color="primary"
-            href={`${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-feature-importance.html`}
-          >
-            <FormattedMessage
-              id="xpack.ml.dataframe.analytics.exploration.featureImportanceDocsLink"
-              defaultMessage="Feature importance docs"
-            />
-          </EuiButtonEmpty>
-        </EuiFlexItem>
-      </EuiFlexGroup>
-      <Chart className="story-chart" size={{ width: '100%', height: 300 }}>
-        <Settings rotation={90} />
-
-        <Axis id="x-axis" title="Feature importance average magnitude" position={Position.Bottom} />
-        <Axis id="y-axis" title="" position={Position.Left} />
-        <BarSeries
-          id="bars"
-          xScaleType={ScaleType.Ordinal}
-          yScaleType={ScaleType.Linear}
-          xAccessor={0}
-          yAccessors={[1]}
-          data={mockData}
-        />
-      </Chart>
-    </EuiPanel>
-  );
-};
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts
deleted file mode 100644
index 5c447b40feb77..0000000000000
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_feature_importance_summary/index.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-export { FeatureImportanceSummary } from './feature_importance_summary';
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
index c97e03b8095a4..84b44ef0d349f 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx
@@ -27,15 +27,9 @@ interface Props {
   jobId: string;
   title: string;
   EvaluatePanel: FC<EvaluatePanelProps>;
-  FeatureImportanceSummaryPanel?: FC<EvaluatePanelProps>;
 }
 
-export const ExplorationPageWrapper: FC<Props> = ({
-  jobId,
-  title,
-  EvaluatePanel,
-  FeatureImportanceSummaryPanel,
-}) => {
+export const ExplorationPageWrapper: FC<Props> = ({ jobId, title, EvaluatePanel }) => {
   const {
     indexPattern,
     isInitialized,
@@ -64,16 +58,6 @@ export const ExplorationPageWrapper: FC<Props> = ({
         <EvaluatePanel jobConfig={jobConfig} jobStatus={jobStatus} searchQuery={searchQuery} />
       )}
       <EuiSpacer />
-      {FeatureImportanceSummaryPanel !== undefined && (
-        <>
-          <FeatureImportanceSummaryPanel
-            jobConfig={jobConfig}
-            jobStatus={jobStatus}
-            searchQuery={searchQuery}
-          />
-          <EuiSpacer />
-        </>
-      )}
       {isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
       {isLoadingJobConfig === false &&
         jobConfig !== undefined &&
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
index c27b95b8e3aae..36d91f6f41d44 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx
@@ -11,7 +11,6 @@ import { i18n } from '@kbn/i18n';
 import { ExplorationPageWrapper } from '../exploration_page_wrapper';
 
 import { EvaluatePanel } from './evaluate_panel';
-import { FeatureImportanceSummary } from '../exploration_feature_importance_summary';
 
 interface Props {
   jobId: string;
@@ -26,7 +25,6 @@ export const RegressionExploration: FC<Props> = ({ jobId }) => {
         values: { jobId },
       })}
       EvaluatePanel={EvaluatePanel}
-      FeatureImportanceSummaryPanel={FeatureImportanceSummary}
     />
   );
 };

From 0c57415a3bc0f968861673d03295364406ce5e35 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 19 Aug 2020 12:13:19 -0500
Subject: [PATCH 17/37] [ML] Update types

---
 .../components/data_grid/data_grid.tsx        |  6 ++-
 .../decision_path_chart.tsx                   | 52 +++++++++++++------
 .../decision_path_popover.tsx                 | 19 ++++---
 .../use_exploration_results.ts                |  2 +-
 4 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index ea02194df46c0..0573f4b4bfa69 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -100,7 +100,11 @@ export const DataGrid: FC<Props> = memo(
           const rowIndex = children?.props?.rowIndex;
           const row = data[rowIndex];
           let predictedValue;
-          if (row && predictionFieldName && row.ml[predictionFieldName] !== undefined) {
+          if (
+            predictionFieldName !== undefined &&
+            row &&
+            row.ml[predictionFieldName] !== undefined
+          ) {
             predictedValue = row.ml[predictionFieldName];
           }
           return (
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 91e3950a0ee04..255c26d85a2b3 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -16,8 +16,9 @@ import {
   Position,
   LineAnnotationDatum,
   PartialTheme,
+  AxisConfig,
+  RecursivePartial,
 } from '@elastic/charts';
-import _ from 'lodash';
 import { EuiIcon } from '@elastic/eui';
 
 const baselineStyle = {
@@ -35,37 +36,54 @@ const baselineStyle = {
   },
 };
 
-const theme: PartialTheme = {
-  axes: {
-    tickLabel: {
-      fontSize: 30,
-    },
+const axes: RecursivePartial<AxisConfig> = {
+  tickLabelStyle: {
+    fontSize: 12,
   },
 };
+const theme: PartialTheme = {
+  axes,
+};
+
+export type DecisionPathPlotData = Array<[string, number, number]>;
 
 interface FeatureImportanceDecisionPathProps {
   baseline?: number;
-  decisionPlotData: LineAnnotationDatum[];
+  decisionPlotData: DecisionPathPlotData | undefined;
   predictedValue?: number | undefined;
 }
 
+const findMaxMin = (data: DecisionPathPlotData, getter: Function): { max: number; min: number } => {
+  let min = Infinity;
+  let max = -Infinity;
+  data.forEach((d) => {
+    const value = getter(d);
+    if (value > max) max = value;
+    if (value < min) min = value;
+  });
+  return { max, min };
+};
+
 export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProps> = ({
   baseline,
   decisionPlotData,
 }) => {
   if (!decisionPlotData) return <div />;
-
   const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
-  let maxDomain = _.maxBy(decisionPlotData, (d) => d[2])[2];
-  let minDomain = _.minBy(decisionPlotData, (d) => d[2])[2];
-  // adjust domain so plot have some space on both sides
-  // and to account for baseline out of range
-  const buffer = Math.abs(maxDomain - minDomain) * 0.1;
-  maxDomain = Math.max(maxDomain, baseline) + buffer;
-  minDomain = Math.min(minDomain, baseline) - buffer;
+  let maxDomain;
+  let minDomain;
+  // if decisionPlotData has calculated cumulative path
+  if (Array.isArray(decisionPlotData) && decisionPlotData.length === 3) {
+    const { max, min } = findMaxMin(decisionPlotData, (d: [string, number, number]) => d[2]);
+    maxDomain = max;
+    minDomain = min;
+    const buffer = Math.abs(maxDomain - minDomain) * 0.1;
+    maxDomain = (typeof baseline === 'number' ? Math.max(maxDomain, baseline) : maxDomain) + buffer;
+    minDomain = (typeof baseline === 'number' ? Math.min(minDomain, baseline) : minDomain) - buffer;
+  }
 
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = decisionPlotData.length > 3 ? 25 : 75;
+  const heightMultiplier = Array.isArray(decisionPlotData) && decisionPlotData.length > 3 ? 25 : 75;
 
   return (
     <>
@@ -103,7 +121,7 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
           xScaleType={ScaleType.Ordinal}
           yScaleType={ScaleType.Linear}
           xAccessor={0}
-          yAccessors={[2]} // if baseline exist then shows the decision path else show in order of descending importance
+          yAccessors={[2]}
           data={decisionPlotData}
         />
       </Chart>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index f15a6ca33ebaf..127320728f45d 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -7,7 +7,7 @@
 import React, { FC, useEffect, useState } from 'react';
 import { EuiTabs, EuiTab, EuiText, EuiLink } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
-import { FeatureImportanceDecisionPath } from './decision_path_chart';
+import { FeatureImportanceDecisionPath, DecisionPathPlotData } from './decision_path_chart';
 import { DecisionPathJSONViewer } from './decision_path_json_viewer';
 import { FeatureImportance } from '../../../../../common/types/feature_importance';
 
@@ -33,7 +33,7 @@ export const useDecisionPathData = ({
   featureImportance,
   predictedValue,
 }: DecisionPathPopoverProps) => {
-  const [decisionPlotData, setDecisionPlotData] = useState();
+  const [decisionPlotData, setDecisionPlotData] = useState<DecisionPathPlotData | undefined>();
 
   useEffect(() => {
     let mappedFeatureImportance: ExtendedFeatureImportance[] = featureImportance;
@@ -42,7 +42,7 @@ export const useDecisionPathData = ({
       absImportance: Math.abs(d[FEATURE_IMPORTANCE]),
     }));
 
-    if (baseline && Number.isFinite(predictedValue)) {
+    if (baseline && predictedValue !== undefined && Number.isFinite(predictedValue)) {
       // get the adjusted importance needed for when # of fields included in c++ analysis != max allowed
       // if num fields included = num features allowed exactly, adjustedImportance should be 0
       const adjustedImportance =
@@ -61,19 +61,19 @@ export const useDecisionPathData = ({
       });
     }
 
-    mappedFeatureImportance = mappedFeatureImportance
+    const finalResult: DecisionPathPlotData = mappedFeatureImportance
       // sort so absolute importance so it goes from bottom (baseline) to top
-      .sort((a, b) => b.absImportance - a.absImportance)
-      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE]]);
+      .sort((a, b) => b.absImportance! - a.absImportance!)
+      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE], NaN]);
 
     // start at the baseline and end at predicted value
     // for regression, cumulativeSum should add up to baseline
     let cumulativeSum = 0;
     for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
-      cumulativeSum += mappedFeatureImportance[i][1];
-      mappedFeatureImportance[i][2] = cumulativeSum;
+      cumulativeSum += finalResult[i][1];
+      finalResult[i][2] = cumulativeSum;
     }
-    setDecisionPlotData(mappedFeatureImportance);
+    setDecisionPlotData(finalResult);
   }, [baseline, featureImportance]);
 
   return { decisionPlotData };
@@ -150,7 +150,6 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
 
           <FeatureImportanceDecisionPath
             baseline={baseline}
-            featureImportance={featureImportance}
             decisionPlotData={decisionPlotData}
             predictedValue={predictedValue}
           />
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 616dd6bba1503..b705b585cbacd 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -115,7 +115,7 @@ export const useExplorationResults = (
     jobConfig?.dest.index,
     JSON.stringify([searchQuery, dataGrid.visibleColumns]),
   ]);
-  const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
+  const predictionFieldName = jobConfig ? getPredictionFieldName(jobConfig.analysis) : undefined;
 
   const getAnalyticsBaseline = useCallback(async () => {
     try {

From 6b525b4d89eac07b3d3262873c873010e712b05e Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Wed, 19 Aug 2020 12:14:33 -0500
Subject: [PATCH 18/37] [ML] Update types for popOverContent

---
 .../public/application/components/data_grid/data_grid.tsx | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 0573f4b4bfa69..ee114d426138a 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -94,7 +94,13 @@ export const DataGrid: FC<Props> = memo(
 
     const popOverContent = useMemo(() => {
       return {
-        featureImportance: ({ cellContentsElement, children }) => {
+        featureImportance: ({
+          cellContentsElement,
+          children,
+        }: {
+          cellContentsElement: any;
+          children: any;
+        }) => {
           const stringContents = cellContentsElement.textContent;
           const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
           const rowIndex = children?.props?.rowIndex;

From c9261a49fd7d046d154568775573a9dd05f932dc Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 20 Aug 2020 14:29:12 -0500
Subject: [PATCH 19/37] [ML] Refactor to support classification DFA

---
 .../ml/common/types/feature_importance.ts     |  15 +-
 .../components/data_grid/data_grid.tsx        |  62 +++----
 .../decision_path_chart.tsx                   |  77 ++++----
 .../decision_path_classification.tsx          | 141 +++++++++++++++
 .../decision_path_popover.tsx                 |  91 +++-------
 .../use_classification_path_data.tsx          | 164 ++++++++++++++++++
 .../exploration_results_table.tsx             |   5 +
 7 files changed, 431 insertions(+), 124 deletions(-)
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx

diff --git a/x-pack/plugins/ml/common/types/feature_importance.ts b/x-pack/plugins/ml/common/types/feature_importance.ts
index e93018253c38c..b44ebcb51c199 100644
--- a/x-pack/plugins/ml/common/types/feature_importance.ts
+++ b/x-pack/plugins/ml/common/types/feature_importance.ts
@@ -4,7 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export interface ClassFeatureImportance {
+  class_name: string;
+  importance: number;
+}
 export interface FeatureImportance {
   feature_name: string;
-  importance: number;
+  importance?: number;
+  classes?: ClassFeatureImportance[];
 }
+
+export interface TopClass {
+  class_name: string;
+  class_probability: number;
+  class_score: number;
+}
+
+export type TopClasses = TopClass[];
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index ee114d426138a..daa106798ebaf 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -25,11 +25,12 @@ import {
 import { CoreSetup } from 'src/core/public';
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms';
 
-import { INDEX_STATUS } from '../../data_frame_analytics/common';
+import { ANALYSIS_CONFIG_TYPE, INDEX_STATUS } from '../../data_frame_analytics/common';
 
 import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
 import { UseIndexDataReturnType } from './types';
 import { DecisionPathPopover } from './feature_importance/decision_path_popover';
+import { TopClasses } from '../../../../common/types/feature_importance';
 
 // TODO Fix row hovering + bar highlighting
 // import { hoveredRow$ } from './column_chart';
@@ -42,6 +43,7 @@ export const DataGridTitle: FC<{ title: string }> = ({ title }) => (
 
 interface PropsWithoutHeader extends UseIndexDataReturnType {
   baseline?: number;
+  analysisType?: ANALYSIS_CONFIG_TYPE;
   dataTestSubj: string;
   toastNotifications: CoreSetup['notifications']['toasts'];
 }
@@ -83,6 +85,7 @@ export const DataGrid: FC<Props> = memo(
       toggleChartVisibility,
       visibleColumns,
       predictionFieldName,
+      analysisType,
     } = props;
     // TODO Fix row hovering + bar highlighting
     // const getRowProps = (item: any) => {
@@ -93,35 +96,36 @@ export const DataGrid: FC<Props> = memo(
     // };
 
     const popOverContent = useMemo(() => {
-      return {
-        featureImportance: ({
-          cellContentsElement,
-          children,
-        }: {
-          cellContentsElement: any;
-          children: any;
-        }) => {
-          const stringContents = cellContentsElement.textContent;
-          const parsedFIArray = stringContents ? JSON.parse(stringContents) : [];
-          const rowIndex = children?.props?.rowIndex;
-          const row = data[rowIndex];
-          let predictedValue;
-          if (
-            predictionFieldName !== undefined &&
-            row &&
-            row.ml[predictionFieldName] !== undefined
-          ) {
-            predictedValue = row.ml[predictionFieldName];
+      return analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION ||
+        analysisType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION
+        ? {
+            featureImportance: ({ children }: { cellContentsElement: any; children: any }) => {
+              const rowIndex = children?.props?.rowIndex;
+              const row = data[rowIndex];
+              const parsedFIArray = row.ml.feature_importance;
+              let predictedValue: string | number | undefined;
+              let topClasses: TopClasses = [];
+              if (
+                predictionFieldName !== undefined &&
+                row &&
+                row.ml[predictionFieldName] !== undefined
+              ) {
+                predictedValue = row.ml[predictionFieldName];
+                topClasses = row.ml.top_classes;
+              }
+
+              return (
+                <DecisionPathPopover
+                  analysisType={analysisType}
+                  predictedValue={predictedValue}
+                  baseline={baseline}
+                  featureImportance={parsedFIArray}
+                  topClasses={topClasses}
+                />
+              );
+            },
           }
-          return (
-            <DecisionPathPopover
-              predictedValue={predictedValue}
-              baseline={baseline}
-              featureImportance={parsedFIArray}
-            />
-          );
-        },
-      };
+        : undefined;
     }, [baseline, data]);
 
     useEffect(() => {
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 255c26d85a2b3..25a80a64f6f4b 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -20,6 +20,9 @@ import {
   RecursivePartial,
 } from '@elastic/charts';
 import { EuiIcon } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
+import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
 
 const baselineStyle = {
   line: {
@@ -45,36 +48,41 @@ const theme: PartialTheme = {
   axes,
 };
 
-export type DecisionPathPlotData = Array<[string, number, number]>;
-
-interface FeatureImportanceDecisionPathProps {
+interface RegressionDecisionPathProps {
   baseline?: number;
-  decisionPlotData: DecisionPathPlotData | undefined;
   predictedValue?: number | undefined;
+  featureImportance: FeatureImportance[];
+  topClasses?: TopClasses;
 }
 
-const findMaxMin = (data: DecisionPathPlotData, getter: Function): { max: number; min: number } => {
-  let min = Infinity;
-  let max = -Infinity;
-  data.forEach((d) => {
-    const value = getter(d);
-    if (value > max) max = value;
-    if (value < min) min = value;
-  });
-  return { max, min };
-};
-
-export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProps> = ({
+export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
   baseline,
-  decisionPlotData,
+  featureImportance,
+  predictedValue,
 }) => {
-  if (!decisionPlotData) return <div />;
-  const baselineData: LineAnnotationDatum[] = [{ dataValue: baseline, details: 'baseline' }];
+  const { decisionPathData } = useDecisionPathData({
+    baseline,
+    featureImportance,
+    predictedValue,
+  });
+  if (!decisionPathData) return <div />;
+  const baselineData: LineAnnotationDatum[] = [
+    {
+      dataValue: baseline,
+      details: i18n.translate(
+        'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathBaselineText',
+        {
+          defaultMessage:
+            'baseline (average of predictions for all data points in the training data set)',
+        }
+      ),
+    },
+  ];
   let maxDomain;
   let minDomain;
-  // if decisionPlotData has calculated cumulative path
-  if (Array.isArray(decisionPlotData) && decisionPlotData.length === 3) {
-    const { max, min } = findMaxMin(decisionPlotData, (d: [string, number, number]) => d[2]);
+  // if decisionPathData has calculated cumulative path
+  if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+    const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
     maxDomain = max;
     minDomain = min;
     const buffer = Math.abs(maxDomain - minDomain) * 0.1;
@@ -83,15 +91,15 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
   }
 
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = Array.isArray(decisionPlotData) && decisionPlotData.length > 3 ? 25 : 75;
+  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 3 ? 40 : 75;
 
   return (
     <>
-      <Chart className="story-chart" size={{ height: decisionPlotData.length * heightMultiplier }}>
+      <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
         <Settings theme={theme} rotation={90} />
         {baseline && (
           <LineAnnotation
-            id="dfa_baseline"
+            id="xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathBaseline"
             domainType={AnnotationDomainTypes.YDomain}
             dataValues={baselineData}
             style={baselineStyle}
@@ -100,10 +108,15 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         )}
 
         <Axis
+          id={'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathXAxis'}
           tickFormat={(d) => `${Number(d).toPrecision(3)}`}
-          title={'Prediction'}
+          title={i18n.translate(
+            'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathXAxisTitle',
+            {
+              defaultMessage: 'Prediction',
+            }
+          )}
           showGridLines={true}
-          id="bottom"
           position={Position.Bottom}
           showOverlappingTicks
           domain={
@@ -117,12 +130,18 @@ export const FeatureImportanceDecisionPath: FC<FeatureImportanceDecisionPathProp
         />
         <Axis showGridLines={true} id="left" position={Position.Left} />
         <LineSeries
-          id="prediction"
+          id="xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathLine"
+          name={i18n.translate(
+            'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathLineTitle',
+            {
+              defaultMessage: 'Prediction',
+            }
+          )}
           xScaleType={ScaleType.Ordinal}
           yScaleType={ScaleType.Linear}
           xAccessor={0}
           yAccessors={[2]}
-          data={decisionPlotData}
+          data={decisionPathData}
         />
       </Chart>
     </>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
new file mode 100644
index 0000000000000..f2d5b8e96a612
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC, useMemo, useState } from 'react';
+import {
+  Chart,
+  Settings,
+  LineSeries,
+  Axis,
+  ScaleType,
+  Position,
+  PartialTheme,
+  AxisConfig,
+  RecursivePartial,
+} from '@elastic/charts';
+import { i18n } from '@kbn/i18n';
+import { EuiHealth, EuiSpacer, EuiSuperSelect, EuiTitle } from '@elastic/eui';
+import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
+import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
+
+const axes: RecursivePartial<AxisConfig> = {
+  tickLabelStyle: {
+    fontSize: 12,
+  },
+};
+const theme: PartialTheme = {
+  axes,
+};
+interface ClassificationDecisionPathProps {
+  predictedValue: string | undefined;
+  featureImportance: FeatureImportance[];
+  topClasses: TopClasses;
+}
+
+export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = ({
+  featureImportance,
+  predictedValue,
+  topClasses,
+}) => {
+  const [currentClass, setCurrentClass] = useState<string>(topClasses[0].class_name);
+  const { decisionPathData } = useDecisionPathData({
+    featureImportance,
+    predictedValue: currentClass,
+  });
+  const options = useMemo(
+    () =>
+      Array.isArray(topClasses) && typeof predictedValue === 'string'
+        ? topClasses.map((c) => ({
+            value: c.class_name,
+            inputDisplay:
+              c.class_name === predictedValue ? (
+                <EuiHealth color="success" style={{ lineHeight: 'inherit' }}>
+                  {c.class_name}
+                </EuiHealth>
+              ) : (
+                c.class_name
+              ),
+          }))
+        : undefined,
+    [topClasses, predictedValue]
+  );
+
+  if (!decisionPathData) return <div />;
+  let maxDomain;
+  let minDomain;
+  // if decisionPathData has calculated cumulative path
+  if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+    const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
+    const buffer = Math.abs(max - min) * 0.1;
+    maxDomain = max + buffer;
+    minDomain = min - buffer;
+  }
+
+  // adjust the height so it's compact for items with more features
+  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 3 ? 35 : 75;
+  return (
+    <>
+      <EuiSpacer size={'xs'} />
+      <EuiTitle size={'xxxs'}>
+        <span>
+          {i18n.translate(
+            'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathClassNameTitle',
+            {
+              defaultMessage: 'Class name',
+            }
+          )}
+        </span>
+      </EuiTitle>
+      {options !== undefined && (
+        <EuiSuperSelect
+          compressed={true}
+          options={options}
+          valueOfSelected={currentClass}
+          onChange={setCurrentClass}
+        />
+      )}
+      <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
+        <Settings theme={theme} rotation={90} />
+        <Axis
+          id={'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathXAxis'}
+          tickFormat={(d) => `${Number(d).toPrecision(3)}`}
+          title={i18n.translate(
+            'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathXAxisTitle',
+            {
+              defaultMessage: 'Prediction',
+            }
+          )}
+          showGridLines={true}
+          position={Position.Bottom}
+          showOverlappingTicks
+          domain={
+            minDomain && maxDomain
+              ? {
+                  min: minDomain,
+                  max: maxDomain,
+                }
+              : undefined
+          }
+        />
+        <Axis showGridLines={true} id="left" position={Position.Left} />
+        <LineSeries
+          id={'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathLine'}
+          name={i18n.translate(
+            'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathLineTitle',
+            {
+              defaultMessage: 'Prediction',
+            }
+          )}
+          xScaleType={ScaleType.Ordinal}
+          yScaleType={ScaleType.Linear}
+          xAccessor={0}
+          yAccessors={[2]}
+          data={decisionPathData}
+        />
+      </Chart>
+    </>
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 127320728f45d..aa11a5061dcee 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -4,17 +4,21 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { FC, useEffect, useState } from 'react';
-import { EuiTabs, EuiTab, EuiText, EuiLink } from '@elastic/eui';
+import React, { FC, useState } from 'react';
+import { EuiLink, EuiTab, EuiTabs, EuiText } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
-import { FeatureImportanceDecisionPath, DecisionPathPlotData } from './decision_path_chart';
+import { RegressionDecisionPath } from './decision_path_chart';
 import { DecisionPathJSONViewer } from './decision_path_json_viewer';
-import { FeatureImportance } from '../../../../../common/types/feature_importance';
+import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
+import { ANALYSIS_CONFIG_TYPE } from '../../../data_frame_analytics/common';
+import { ClassificationDecisionPath } from './decision_path_classification';
 
 interface DecisionPathPopoverProps {
-  baseline?: number;
   featureImportance: FeatureImportance[];
-  predictedValue?: number | undefined;
+  analysisType: ANALYSIS_CONFIG_TYPE;
+  baseline?: number;
+  predictedValue?: number | string | undefined;
+  topClasses?: TopClasses;
 }
 
 enum DECISION_PATH_TABS {
@@ -22,69 +26,18 @@ enum DECISION_PATH_TABS {
   JSON = 'decision_path_json',
 }
 
-const FEATURE_NAME = 'feature_name';
-const FEATURE_IMPORTANCE = 'importance';
-
 export interface ExtendedFeatureImportance extends FeatureImportance {
   absImportance?: number;
 }
-export const useDecisionPathData = ({
-  baseline,
-  featureImportance,
-  predictedValue,
-}: DecisionPathPopoverProps) => {
-  const [decisionPlotData, setDecisionPlotData] = useState<DecisionPathPlotData | undefined>();
-
-  useEffect(() => {
-    let mappedFeatureImportance: ExtendedFeatureImportance[] = featureImportance;
-    mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
-      ...d,
-      absImportance: Math.abs(d[FEATURE_IMPORTANCE]),
-    }));
-
-    if (baseline && predictedValue !== undefined && Number.isFinite(predictedValue)) {
-      // get the adjusted importance needed for when # of fields included in c++ analysis != max allowed
-      // if num fields included = num features allowed exactly, adjustedImportance should be 0
-      const adjustedImportance =
-        predictedValue -
-        mappedFeatureImportance.reduce(
-          (accumulator, currentValue) => accumulator + currentValue.importance,
-          0
-        ) -
-        baseline;
 
-      // get the absolute absImportance of the importance value to the baseline for sorting
-      mappedFeatureImportance.push({
-        [FEATURE_NAME]: 'other',
-        [FEATURE_IMPORTANCE]: baseline + adjustedImportance,
-        absImportance: 0,
-      });
-    }
-
-    const finalResult: DecisionPathPlotData = mappedFeatureImportance
-      // sort so absolute importance so it goes from bottom (baseline) to top
-      .sort((a, b) => b.absImportance! - a.absImportance!)
-      .map((d) => [d[FEATURE_NAME], d[FEATURE_IMPORTANCE], NaN]);
-
-    // start at the baseline and end at predicted value
-    // for regression, cumulativeSum should add up to baseline
-    let cumulativeSum = 0;
-    for (let i = mappedFeatureImportance.length - 1; i >= 0; i--) {
-      cumulativeSum += finalResult[i][1];
-      finalResult[i][2] = cumulativeSum;
-    }
-    setDecisionPlotData(finalResult);
-  }, [baseline, featureImportance]);
-
-  return { decisionPlotData };
-};
 export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   baseline,
   featureImportance,
   predictedValue,
+  topClasses,
+  analysisType,
 }) => {
   const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
-  const { decisionPlotData } = useDecisionPathData({ baseline, featureImportance, predictedValue });
 
   if (featureImportance.length < 2) {
     return <DecisionPathJSONViewer featureImportance={featureImportance} />;
@@ -147,12 +100,20 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
               }}
             />
           </EuiText>
-
-          <FeatureImportanceDecisionPath
-            baseline={baseline}
-            decisionPlotData={decisionPlotData}
-            predictedValue={predictedValue}
-          />
+          {analysisType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION && (
+            <ClassificationDecisionPath
+              featureImportance={featureImportance}
+              topClasses={topClasses as TopClasses}
+              predictedValue={predictedValue as string}
+            />
+          )}
+          {analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION && (
+            <RegressionDecisionPath
+              featureImportance={featureImportance}
+              baseline={baseline}
+              predictedValue={predictedValue as number}
+            />
+          )}
         </>
       )}
       {selectedTabId === DECISION_PATH_TABS.JSON && (
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
new file mode 100644
index 0000000000000..f5eeb1468ff99
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useEffect, useState } from 'react';
+import { i18n } from '@kbn/i18n';
+import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
+import { ExtendedFeatureImportance } from './decision_path_popover';
+
+export type DecisionPathPlotData = Array<[string, number, number]>;
+
+interface UseDecisionPathDataParams {
+  featureImportance: FeatureImportance[];
+  baseline?: number;
+  predictedValue?: string | number | undefined;
+  topClasses?: TopClasses;
+}
+
+interface RegressionDecisionPathProps {
+  baseline?: number;
+  predictedValue?: number | undefined;
+  featureImportance: FeatureImportance[];
+  topClasses?: TopClasses;
+}
+const FEATURE_NAME = 'feature_name';
+const FEATURE_IMPORTANCE = 'importance';
+
+export const useDecisionPathData = ({
+  baseline,
+  featureImportance,
+  predictedValue,
+}: UseDecisionPathDataParams): { decisionPathData: DecisionPathPlotData | undefined } => {
+  const [decisionPathData, setDecisionPlotData] = useState<DecisionPathPlotData | undefined>();
+
+  useEffect(() => {
+    const result = baseline
+      ? buildRegressionDecisionPathData({
+          baseline,
+          featureImportance,
+          predictedValue: predictedValue as number | undefined,
+        })
+      : buildClassificationDecisionPathData({
+          featureImportance,
+          currentClass: predictedValue as string | undefined,
+        });
+
+    setDecisionPlotData(result);
+  }, [baseline, featureImportance, predictedValue]);
+
+  return { decisionPathData };
+};
+
+export const buildDecisionPathData = (featureImportance: ExtendedFeatureImportance[]) => {
+  const finalResult: DecisionPathPlotData = featureImportance
+    // sort so absolute importance so it goes from bottom (baseline) to top
+    .sort(
+      (a: ExtendedFeatureImportance, b: ExtendedFeatureImportance) =>
+        b.absImportance! - a.absImportance!
+    )
+    .map((d) => [d[FEATURE_NAME] as string, d[FEATURE_IMPORTANCE] as number, NaN]);
+
+  // start at the baseline and end at predicted value
+  // for regression, cumulativeSum should add up to baseline
+  let cumulativeSum = 0;
+  for (let i = featureImportance.length - 1; i >= 0; i--) {
+    cumulativeSum += finalResult[i][1];
+    finalResult[i][2] = cumulativeSum;
+  }
+  return finalResult;
+};
+export const buildRegressionDecisionPathData = ({
+  baseline,
+  featureImportance,
+  predictedValue,
+}: RegressionDecisionPathProps): DecisionPathPlotData | undefined => {
+  let mappedFeatureImportance: ExtendedFeatureImportance[] = featureImportance;
+  mappedFeatureImportance = mappedFeatureImportance.map((d) => ({
+    ...d,
+    absImportance: Math.abs(d[FEATURE_IMPORTANCE] as number),
+  }));
+
+  if (baseline && predictedValue !== undefined && Number.isFinite(predictedValue)) {
+    // get the adjusted importance needed for when # of fields included in c++ analysis != max allowed
+    // if num fields included = num features allowed exactly, adjustedImportance should be 0
+    const adjustedImportance =
+      predictedValue -
+      mappedFeatureImportance.reduce(
+        (accumulator, currentValue) => accumulator + currentValue.importance!,
+        0
+      ) -
+      baseline;
+
+    mappedFeatureImportance.push({
+      [FEATURE_NAME]: i18n.translate(
+        'xpack.ml.dataframe.analytics.decisionPathFeatureBaselineTitle',
+        {
+          defaultMessage: 'baseline',
+        }
+      ),
+      [FEATURE_IMPORTANCE]: baseline,
+      absImportance: -1,
+    });
+
+    // get the absolute absImportance of the importance value to the baseline for sorting
+    mappedFeatureImportance.push({
+      [FEATURE_NAME]: i18n.translate('xpack.ml.dataframe.analytics.decisionPathFeatureOtherTitle', {
+        defaultMessage: 'other',
+      }),
+      [FEATURE_IMPORTANCE]: baseline + adjustedImportance,
+      absImportance: 0, // arbitrary importance so this will be of higher importance than baseline
+    });
+  }
+  const filteredFeatureImportance = featureImportance.filter(
+    (f) => f !== undefined
+  ) as ExtendedFeatureImportance[];
+
+  return buildDecisionPathData(filteredFeatureImportance);
+};
+
+export const buildClassificationDecisionPathData = ({
+  featureImportance,
+  currentClass,
+}: {
+  featureImportance: FeatureImportance[];
+  currentClass: string | undefined;
+}): DecisionPathPlotData | undefined => {
+  if (currentClass === undefined) return [];
+  const mappedFeatureImportance: Array<
+    ExtendedFeatureImportance | undefined
+  > = featureImportance.map((feature) => {
+    const classFeatureImportance = Array.isArray(feature.classes)
+      ? feature.classes.find((c) => c.class_name === currentClass)
+      : feature;
+    if (classFeatureImportance && typeof classFeatureImportance[FEATURE_IMPORTANCE] === 'number') {
+      return {
+        [FEATURE_NAME]: feature[FEATURE_NAME],
+        [FEATURE_IMPORTANCE]: classFeatureImportance[FEATURE_IMPORTANCE],
+        absImportance: Math.abs(classFeatureImportance[FEATURE_IMPORTANCE] as number),
+      };
+    }
+    return undefined;
+  });
+  const filteredFeatureImportance = mappedFeatureImportance.filter(
+    (f) => f !== undefined
+  ) as ExtendedFeatureImportance[];
+
+  return buildDecisionPathData(filteredFeatureImportance);
+};
+
+export const findMaxMin = (
+  data: DecisionPathPlotData,
+  getter: Function
+): { max: number; min: number } => {
+  let min = Infinity;
+  let max = -Infinity;
+  data.forEach((d) => {
+    const value = getter(d);
+    if (value > max) max = value;
+    if (value < min) min = value;
+  });
+  return { max, min };
+};
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
index 1a1b1ac508951..eea579ef1d064 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/exploration_results_table.tsx
@@ -28,6 +28,8 @@ import {
   INDEX_STATUS,
   SEARCH_SIZE,
   defaultSearchQuery,
+  getAnalysisType,
+  ANALYSIS_CONFIG_TYPE,
 } from '../../../../common';
 import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/use_columns';
 import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
@@ -82,6 +84,8 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
       setEvaluateSearchQuery(searchQuery);
     }, [JSON.stringify(searchQuery)]);
 
+    const analysisType = getAnalysisType(jobConfig.analysis);
+
     const classificationData = useExplorationResults(
       indexPattern,
       jobConfig,
@@ -191,6 +195,7 @@ export const ExplorationResultsTable: FC<Props> = React.memo(
                 {...classificationData}
                 dataTestSubj="mlExplorationDataGrid"
                 toastNotifications={getToastNotifications()}
+                analysisType={(analysisType as unknown) as ANALYSIS_CONFIG_TYPE}
               />
             </EuiFlexItem>
           </EuiFlexGroup>

From 38a5e48754fcd459418815c4c122bd65207d1495 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 20 Aug 2020 14:44:09 -0500
Subject: [PATCH 20/37] [ML] Update filteredFeatureImportance

---
 .../feature_importance/use_classification_path_data.tsx         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
index f5eeb1468ff99..f16b6dd368bd3 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -112,7 +112,7 @@ export const buildRegressionDecisionPathData = ({
       absImportance: 0, // arbitrary importance so this will be of higher importance than baseline
     });
   }
-  const filteredFeatureImportance = featureImportance.filter(
+  const filteredFeatureImportance = mappedFeatureImportance.filter(
     (f) => f !== undefined
   ) as ExtendedFeatureImportance[];
 

From c6cf1818492e78872ac1719f18aa0f33d9bba136 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 20 Aug 2020 15:59:31 -0500
Subject: [PATCH 21/37] [ML] Add improvement to decision path chart

- Add clearer message with prediction field name
- Only show residual feature importance if needed
- Round baseline data point to 3 sigfig
- Refactor separate DecisionPathChart
---
 .../components/data_grid/data_grid.tsx        |   3 +
 .../decision_path_chart.tsx                   | 162 ++++++++----------
 .../decision_path_classification.tsx          |  91 +++-------
 .../decision_path_popover.tsx                 |   9 +-
 .../decision_path_regression.tsx              |  59 +++++++
 .../use_classification_path_data.tsx          |  21 ++-
 6 files changed, 176 insertions(+), 169 deletions(-)
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index daa106798ebaf..a736b0e9ce011 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -121,6 +121,9 @@ export const DataGrid: FC<Props> = memo(
                   baseline={baseline}
                   featureImportance={parsedFIArray}
                   topClasses={topClasses}
+                  predictionFieldName={
+                    predictionFieldName ? predictionFieldName.replace('_prediction', '') : undefined
+                  }
                 />
               );
             },
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 25a80a64f6f4b..29d683b7b5174 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -4,25 +4,26 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { FC } from 'react';
+// adjust the height so it's compact for items with more features
 import {
-  Chart,
-  Settings,
-  LineAnnotation,
   AnnotationDomainTypes,
-  LineSeries,
   Axis,
-  ScaleType,
-  Position,
+  AxisConfig,
+  Chart,
+  LineAnnotation,
   LineAnnotationDatum,
+  LineSeries,
   PartialTheme,
-  AxisConfig,
+  Position,
   RecursivePartial,
+  ScaleType,
+  Settings,
 } from '@elastic/charts';
 import { EuiIcon } from '@elastic/eui';
+
+import React from 'react';
 import { i18n } from '@kbn/i18n';
-import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
-import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
+import { DecisionPathPlotData } from './use_classification_path_data';
 
 const baselineStyle = {
   line: {
@@ -48,29 +49,27 @@ const theme: PartialTheme = {
   axes,
 };
 
-interface RegressionDecisionPathProps {
+interface DecisionPathChartProps {
+  decisionPathData: DecisionPathPlotData;
+  predictionFieldName?: string;
   baseline?: number;
-  predictedValue?: number | undefined;
-  featureImportance: FeatureImportance[];
-  topClasses?: TopClasses;
+  minDomain: number | undefined;
+  maxDomain: number | undefined;
 }
 
-export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
+export const DecisionPathChart = ({
+  decisionPathData,
+  predictionFieldName,
+  minDomain,
+  maxDomain,
   baseline,
-  featureImportance,
-  predictedValue,
-}) => {
-  const { decisionPathData } = useDecisionPathData({
-    baseline,
-    featureImportance,
-    predictedValue,
-  });
-  if (!decisionPathData) return <div />;
+}: DecisionPathChartProps) => {
+  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 4 ? 30 : 75;
   const baselineData: LineAnnotationDatum[] = [
     {
-      dataValue: baseline,
+      dataValue: baseline ? parseFloat(baseline.toPrecision(3)) : undefined,
       details: i18n.translate(
-        'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathBaselineText',
+        'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText',
         {
           defaultMessage:
             'baseline (average of predictions for all data points in the training data set)',
@@ -78,72 +77,57 @@ export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
       ),
     },
   ];
-  let maxDomain;
-  let minDomain;
-  // if decisionPathData has calculated cumulative path
-  if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
-    const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
-    maxDomain = max;
-    minDomain = min;
-    const buffer = Math.abs(maxDomain - minDomain) * 0.1;
-    maxDomain = (typeof baseline === 'number' ? Math.max(maxDomain, baseline) : maxDomain) + buffer;
-    minDomain = (typeof baseline === 'number' ? Math.min(minDomain, baseline) : minDomain) - buffer;
-  }
-
-  // adjust the height so it's compact for items with more features
-  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 3 ? 40 : 75;
 
   return (
-    <>
-      <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
-        <Settings theme={theme} rotation={90} />
-        {baseline && (
-          <LineAnnotation
-            id="xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathBaseline"
-            domainType={AnnotationDomainTypes.YDomain}
-            dataValues={baselineData}
-            style={baselineStyle}
-            marker={<EuiIcon type={'annotation'} />}
-          />
-        )}
+    <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
+      <Settings theme={theme} rotation={90} />
+      {baseline && (
+        <LineAnnotation
+          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathBaseline"
+          domainType={AnnotationDomainTypes.YDomain}
+          dataValues={baselineData}
+          style={baselineStyle}
+          marker={<EuiIcon type={'annotation'} />}
+        />
+      )}
 
-        <Axis
-          id={'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathXAxis'}
-          tickFormat={(d) => `${Number(d).toPrecision(3)}`}
-          title={i18n.translate(
-            'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathXAxisTitle',
-            {
-              defaultMessage: 'Prediction',
-            }
-          )}
-          showGridLines={true}
-          position={Position.Bottom}
-          showOverlappingTicks
-          domain={
-            minDomain && maxDomain
-              ? {
-                  min: minDomain,
-                  max: maxDomain,
-                }
-              : undefined
+      <Axis
+        id={'xpack.ml.dataframe.analytics.explorationResults.decisionPathXAxis'}
+        tickFormat={(d) => `${Number(d).toPrecision(3)}`}
+        title={i18n.translate(
+          'xpack.ml.dataframe.analytics.explorationResults.decisionPathXAxisTitle',
+          {
+            defaultMessage: "Prediction for '{predictionFieldName}'",
+            values: { predictionFieldName },
           }
-        />
-        <Axis showGridLines={true} id="left" position={Position.Left} />
-        <LineSeries
-          id="xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathLine"
-          name={i18n.translate(
-            'xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathLineTitle',
-            {
-              defaultMessage: 'Prediction',
-            }
-          )}
-          xScaleType={ScaleType.Ordinal}
-          yScaleType={ScaleType.Linear}
-          xAccessor={0}
-          yAccessors={[2]}
-          data={decisionPathData}
-        />
-      </Chart>
-    </>
+        )}
+        showGridLines={true}
+        position={Position.Bottom}
+        showOverlappingTicks
+        domain={
+          minDomain && maxDomain
+            ? {
+                min: minDomain,
+                max: maxDomain,
+              }
+            : undefined
+        }
+      />
+      <Axis showGridLines={true} id="left" position={Position.Left} />
+      <LineSeries
+        id={'xpack.ml.dataframe.analytics.explorationResults.decisionPathLine'}
+        name={i18n.translate(
+          'xpack.ml.dataframe.analytics.explorationResults.decisionPathLineTitle',
+          {
+            defaultMessage: 'Prediction',
+          }
+        )}
+        xScaleType={ScaleType.Ordinal}
+        yScaleType={ScaleType.Linear}
+        xAccessor={0}
+        yAccessors={[2]}
+        data={decisionPathData}
+      />
+    </Chart>
   );
 };
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
index f2d5b8e96a612..051974946a9f2 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -5,32 +5,14 @@
  */
 
 import React, { FC, useMemo, useState } from 'react';
-import {
-  Chart,
-  Settings,
-  LineSeries,
-  Axis,
-  ScaleType,
-  Position,
-  PartialTheme,
-  AxisConfig,
-  RecursivePartial,
-} from '@elastic/charts';
 import { i18n } from '@kbn/i18n';
 import { EuiHealth, EuiSpacer, EuiSuperSelect, EuiTitle } from '@elastic/eui';
 import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
-
-const axes: RecursivePartial<AxisConfig> = {
-  tickLabelStyle: {
-    fontSize: 12,
-  },
-};
-const theme: PartialTheme = {
-  axes,
-};
+import { DecisionPathChart } from './decision_path_chart';
 interface ClassificationDecisionPathProps {
   predictedValue: string | undefined;
+  predictionFieldName?: string;
   featureImportance: FeatureImportance[];
   topClasses: TopClasses;
 }
@@ -39,6 +21,7 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
   featureImportance,
   predictedValue,
   topClasses,
+  predictionFieldName,
 }) => {
   const [currentClass, setCurrentClass] = useState<string>(topClasses[0].class_name);
   const { decisionPathData } = useDecisionPathData({
@@ -62,20 +45,21 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
         : undefined,
     [topClasses, predictedValue]
   );
+  const domain = useMemo(() => {
+    let maxDomain;
+    let minDomain;
+    // if decisionPathData has calculated cumulative path
+    if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+      const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
+      const buffer = Math.abs(max - min) * 0.1;
+      maxDomain = max + buffer;
+      minDomain = min - buffer;
+    }
+    return { maxDomain, minDomain };
+  }, [decisionPathData]);
 
   if (!decisionPathData) return <div />;
-  let maxDomain;
-  let minDomain;
-  // if decisionPathData has calculated cumulative path
-  if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
-    const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
-    const buffer = Math.abs(max - min) * 0.1;
-    maxDomain = max + buffer;
-    minDomain = min - buffer;
-  }
 
-  // adjust the height so it's compact for items with more features
-  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 3 ? 35 : 75;
   return (
     <>
       <EuiSpacer size={'xs'} />
@@ -97,45 +81,12 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
           onChange={setCurrentClass}
         />
       )}
-      <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
-        <Settings theme={theme} rotation={90} />
-        <Axis
-          id={'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathXAxis'}
-          tickFormat={(d) => `${Number(d).toPrecision(3)}`}
-          title={i18n.translate(
-            'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathXAxisTitle',
-            {
-              defaultMessage: 'Prediction',
-            }
-          )}
-          showGridLines={true}
-          position={Position.Bottom}
-          showOverlappingTicks
-          domain={
-            minDomain && maxDomain
-              ? {
-                  min: minDomain,
-                  max: maxDomain,
-                }
-              : undefined
-          }
-        />
-        <Axis showGridLines={true} id="left" position={Position.Left} />
-        <LineSeries
-          id={'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathLine'}
-          name={i18n.translate(
-            'xpack.ml.dataframe.analytics.explorationResults.classificationDecisionPathLineTitle',
-            {
-              defaultMessage: 'Prediction',
-            }
-          )}
-          xScaleType={ScaleType.Ordinal}
-          yScaleType={ScaleType.Linear}
-          xAccessor={0}
-          yAccessors={[2]}
-          data={decisionPathData}
-        />
-      </Chart>
+      <DecisionPathChart
+        decisionPathData={decisionPathData}
+        predictionFieldName={predictionFieldName}
+        minDomain={domain.minDomain}
+        maxDomain={domain.maxDomain}
+      />
     </>
   );
 };
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index aa11a5061dcee..738c685de632b 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -7,7 +7,7 @@
 import React, { FC, useState } from 'react';
 import { EuiLink, EuiTab, EuiTabs, EuiText } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
-import { RegressionDecisionPath } from './decision_path_chart';
+import { RegressionDecisionPath } from './decision_path_regression';
 import { DecisionPathJSONViewer } from './decision_path_json_viewer';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { ANALYSIS_CONFIG_TYPE } from '../../../data_frame_analytics/common';
@@ -16,6 +16,7 @@ import { ClassificationDecisionPath } from './decision_path_classification';
 interface DecisionPathPopoverProps {
   featureImportance: FeatureImportance[];
   analysisType: ANALYSIS_CONFIG_TYPE;
+  predictionFieldName?: string;
   baseline?: number;
   predictedValue?: number | string | undefined;
   topClasses?: TopClasses;
@@ -36,6 +37,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   predictedValue,
   topClasses,
   analysisType,
+  predictionFieldName,
 }) => {
   const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
 
@@ -84,8 +86,9 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
           <EuiText size={'xs'} color="subdued" style={{ paddingTop: 5 }}>
             <FormattedMessage
               id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotHelpText"
-              defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted values."
+              defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted at the predicted value for '{predictionFieldName}'."
               values={{
+                predictionFieldName,
                 linkedFeatureImportanceValues: (
                   <EuiLink
                     href="https://www.elastic.co/guide/en/machine-learning/7.8/ml-feature-importance.html"
@@ -105,6 +108,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
               featureImportance={featureImportance}
               topClasses={topClasses as TopClasses}
               predictedValue={predictedValue as string}
+              predictionFieldName={predictionFieldName}
             />
           )}
           {analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION && (
@@ -112,6 +116,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
               featureImportance={featureImportance}
               baseline={baseline}
               predictedValue={predictedValue as number}
+              predictionFieldName={predictionFieldName}
             />
           )}
         </>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
new file mode 100644
index 0000000000000..0cac9e5706556
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC, useMemo } from 'react';
+import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
+import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
+import { DecisionPathChart } from './decision_path_chart';
+
+interface RegressionDecisionPathProps {
+  predictionFieldName?: string;
+  baseline?: number;
+  predictedValue?: number | undefined;
+  featureImportance: FeatureImportance[];
+  topClasses?: TopClasses;
+}
+
+export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
+  baseline,
+  featureImportance,
+  predictedValue,
+  predictionFieldName,
+}) => {
+  const { decisionPathData } = useDecisionPathData({
+    baseline,
+    featureImportance,
+    predictedValue,
+  });
+  const domain = useMemo(() => {
+    let maxDomain;
+    let minDomain;
+    // if decisionPathData has calculated cumulative path
+    if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+      const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
+      maxDomain = max;
+      minDomain = min;
+      const buffer = Math.abs(maxDomain - minDomain) * 0.1;
+      maxDomain =
+        (typeof baseline === 'number' ? Math.max(maxDomain, baseline) : maxDomain) + buffer;
+      minDomain =
+        (typeof baseline === 'number' ? Math.min(minDomain, baseline) : minDomain) - buffer;
+    }
+    return { maxDomain, minDomain };
+  }, [decisionPathData, baseline]);
+
+  if (!decisionPathData) return <div />;
+
+  return (
+    <DecisionPathChart
+      decisionPathData={decisionPathData}
+      predictionFieldName={predictionFieldName}
+      minDomain={domain.minDomain}
+      maxDomain={domain.maxDomain}
+      baseline={baseline}
+    />
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
index f16b6dd368bd3..b06516e4d868e 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -103,14 +103,19 @@ export const buildRegressionDecisionPathData = ({
       absImportance: -1,
     });
 
-    // get the absolute absImportance of the importance value to the baseline for sorting
-    mappedFeatureImportance.push({
-      [FEATURE_NAME]: i18n.translate('xpack.ml.dataframe.analytics.decisionPathFeatureOtherTitle', {
-        defaultMessage: 'other',
-      }),
-      [FEATURE_IMPORTANCE]: baseline + adjustedImportance,
-      absImportance: 0, // arbitrary importance so this will be of higher importance than baseline
-    });
+    // if the difference is small enough then no need to plot the residual feature importance
+    if (adjustedImportance > 1e-5) {
+      mappedFeatureImportance.push({
+        [FEATURE_NAME]: i18n.translate(
+          'xpack.ml.dataframe.analytics.decisionPathFeatureOtherTitle',
+          {
+            defaultMessage: 'other',
+          }
+        ),
+        [FEATURE_IMPORTANCE]: adjustedImportance,
+        absImportance: 0, // arbitrary importance so this will be of higher importance than baseline
+      });
+    }
   }
   const filteredFeatureImportance = mappedFeatureImportance.filter(
     (f) => f !== undefined

From 175bfd4f22e2e9abae8432ba6cc2cc3c6d8b5f5b Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 20 Aug 2020 17:46:46 -0500
Subject: [PATCH 22/37] [ML] Refactor feature_importance to have own model &
 add error handling

---
 .../ml/common/types/data_frame_analytics.ts   |  6 ++
 .../plugins/ml/common/util/analytics_utils.ts | 46 +++++++++++++++
 .../decision_path_chart.tsx                   | 32 +++++-----
 .../decision_path_popover.tsx                 |  4 +-
 .../decision_path_regression.tsx              | 32 +++++++---
 .../data_frame_analytics/common/analytics.ts  | 56 +++++-------------
 .../use_exploration_results.ts                | 21 ++++---
 .../ml_api_service/data_frame_analytics.ts    |  6 +-
 .../feature_importance.ts                     | 59 +++++++++++++++++++
 .../ml/server/routes/data_frame_analytics.ts  | 44 ++++----------
 10 files changed, 197 insertions(+), 109 deletions(-)
 create mode 100644 x-pack/plugins/ml/common/util/analytics_utils.ts
 create mode 100644 x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts

diff --git a/x-pack/plugins/ml/common/types/data_frame_analytics.ts b/x-pack/plugins/ml/common/types/data_frame_analytics.ts
index f0aac75047585..60d2ca63dda59 100644
--- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts
@@ -79,3 +79,9 @@ export interface DataFrameAnalyticsConfig {
   version: string;
   allow_lazy_start?: boolean;
 }
+
+export enum ANALYSIS_CONFIG_TYPE {
+  OUTLIER_DETECTION = 'outlier_detection',
+  REGRESSION = 'regression',
+  CLASSIFICATION = 'classification',
+}
diff --git a/x-pack/plugins/ml/common/util/analytics_utils.ts b/x-pack/plugins/ml/common/util/analytics_utils.ts
new file mode 100644
index 0000000000000..6cafbafc528af
--- /dev/null
+++ b/x-pack/plugins/ml/common/util/analytics_utils.ts
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  AnalysisConfig,
+  ClassificationAnalysis,
+  OutlierAnalysis,
+  RegressionAnalysis,
+  ANALYSIS_CONFIG_TYPE,
+} from '../types/data_frame_analytics';
+
+export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => {
+  const keys = Object.keys(arg);
+  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION;
+};
+
+export const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => {
+  const keys = Object.keys(arg);
+  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION;
+};
+
+export const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => {
+  const keys = Object.keys(arg);
+  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
+};
+
+export const getPredictionFieldName = (
+  analysis: AnalysisConfig
+):
+  | RegressionAnalysis['regression']['prediction_field_name']
+  | ClassificationAnalysis['classification']['prediction_field_name'] => {
+  // If undefined will be defaulted to dependent_variable when config is created
+  let predictionFieldName;
+  if (isRegressionAnalysis(analysis) && analysis.regression.prediction_field_name !== undefined) {
+    predictionFieldName = analysis.regression.prediction_field_name;
+  } else if (
+    isClassificationAnalysis(analysis) &&
+    analysis.classification.prediction_field_name !== undefined
+  ) {
+    predictionFieldName = analysis.classification.prediction_field_name;
+  }
+  return predictionFieldName;
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 29d683b7b5174..2dbfccfffdcd5 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -21,7 +21,7 @@ import {
 } from '@elastic/charts';
 import { EuiIcon } from '@elastic/eui';
 
-import React from 'react';
+import React, { useCallback, useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
 import { DecisionPathPlotData } from './use_classification_path_data';
 
@@ -65,18 +65,22 @@ export const DecisionPathChart = ({
   baseline,
 }: DecisionPathChartProps) => {
   const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 4 ? 30 : 75;
-  const baselineData: LineAnnotationDatum[] = [
-    {
-      dataValue: baseline ? parseFloat(baseline.toPrecision(3)) : undefined,
-      details: i18n.translate(
-        'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText',
-        {
-          defaultMessage:
-            'baseline (average of predictions for all data points in the training data set)',
-        }
-      ),
-    },
-  ];
+  const baselineData: LineAnnotationDatum[] = useMemo(
+    () => [
+      {
+        dataValue: baseline ? parseFloat(baseline.toPrecision(3)) : undefined,
+        details: i18n.translate(
+          'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText',
+          {
+            defaultMessage:
+              'baseline (average of predictions for all data points in the training data set)',
+          }
+        ),
+      },
+    ],
+    [baseline]
+  );
+  const tickFormatter = useCallback((d) => `${Number(d).toPrecision(3)}`, []);
 
   return (
     <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
@@ -93,7 +97,7 @@ export const DecisionPathChart = ({
 
       <Axis
         id={'xpack.ml.dataframe.analytics.explorationResults.decisionPathXAxis'}
-        tickFormat={(d) => `${Number(d).toPrecision(3)}`}
+        tickFormat={tickFormatter}
         title={i18n.translate(
           'xpack.ml.dataframe.analytics.explorationResults.decisionPathXAxisTitle',
           {
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 738c685de632b..84991b803b467 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -50,7 +50,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
       id: DECISION_PATH_TABS.CHART,
       name: (
         <FormattedMessage
-          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlot"
+          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotTab"
           defaultMessage="Decision plot"
         />
       ),
@@ -59,7 +59,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
       id: DECISION_PATH_TABS.JSON,
       name: (
         <FormattedMessage
-          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathJSON"
+          id="xpack.ml.dataframe.analytics.explorationResults.decisionPathJSONTab"
           defaultMessage="JSON"
         />
       ),
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
index 0cac9e5706556..a0b39ee60da37 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
@@ -5,6 +5,8 @@
  */
 
 import React, { FC, useMemo } from 'react';
+import { EuiCallOut } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
 import { DecisionPathChart } from './decision_path_chart';
@@ -48,12 +50,28 @@ export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
   if (!decisionPathData) return <div />;
 
   return (
-    <DecisionPathChart
-      decisionPathData={decisionPathData}
-      predictionFieldName={predictionFieldName}
-      minDomain={domain.minDomain}
-      maxDomain={domain.maxDomain}
-      baseline={baseline}
-    />
+    <>
+      {baseline === undefined && (
+        <EuiCallOut
+          size={'s'}
+          heading={'p'}
+          title={
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.explorationResults.missingBaselineCallout"
+              defaultMessage="Unable to calculate baseline value, which might result in a shifted decision path."
+            />
+          }
+          color="warning"
+          iconType="alert"
+        />
+      )}
+      <DecisionPathChart
+        decisionPathData={decisionPathData}
+        predictionFieldName={predictionFieldName}
+        minDomain={domain.minDomain}
+        maxDomain={domain.maxDomain}
+        baseline={baseline}
+      />
+    </>
   );
 };
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
index 8ad861e616b7a..e6b9197da9099 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
@@ -15,18 +15,17 @@ import { SavedSearchQuery } from '../../contexts/ml';
 import {
   AnalysisConfig,
   ClassificationAnalysis,
-  OutlierAnalysis,
   RegressionAnalysis,
+  ANALYSIS_CONFIG_TYPE,
 } from '../../../../common/types/data_frame_analytics';
-
+import {
+  isOutlierAnalysis,
+  isRegressionAnalysis,
+  isClassificationAnalysis,
+  getPredictionFieldName,
+} from '../../../../common/util/analytics_utils';
 export type IndexPattern = string;
 
-export enum ANALYSIS_CONFIG_TYPE {
-  OUTLIER_DETECTION = 'outlier_detection',
-  REGRESSION = 'regression',
-  CLASSIFICATION = 'classification',
-}
-
 export enum ANALYSIS_ADVANCED_FIELDS {
   ETA = 'eta',
   FEATURE_BAG_FRACTION = 'feature_bag_fraction',
@@ -190,24 +189,6 @@ export const getTrainingPercent = (
   return trainingPercent;
 };
 
-export const getPredictionFieldName = (
-  analysis: AnalysisConfig
-):
-  | RegressionAnalysis['regression']['prediction_field_name']
-  | ClassificationAnalysis['classification']['prediction_field_name'] => {
-  // If undefined will be defaulted to dependent_variable when config is created
-  let predictionFieldName;
-  if (isRegressionAnalysis(analysis) && analysis.regression.prediction_field_name !== undefined) {
-    predictionFieldName = analysis.regression.prediction_field_name;
-  } else if (
-    isClassificationAnalysis(analysis) &&
-    analysis.classification.prediction_field_name !== undefined
-  ) {
-    predictionFieldName = analysis.classification.prediction_field_name;
-  }
-  return predictionFieldName;
-};
-
 export const getNumTopClasses = (
   analysis: AnalysisConfig
 ): ClassificationAnalysis['classification']['num_top_classes'] => {
@@ -252,21 +233,6 @@ export const getPredictedFieldName = (
   return predictedField;
 };
 
-export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => {
-  const keys = Object.keys(arg);
-  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION;
-};
-
-export const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => {
-  const keys = Object.keys(arg);
-  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION;
-};
-
-export const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => {
-  const keys = Object.keys(arg);
-  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
-};
-
 export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuery => {
   if (arg === undefined) return false;
   const keys = Object.keys(arg);
@@ -607,3 +573,11 @@ export const loadDocsCount = async ({
     };
   }
 };
+
+export {
+  isOutlierAnalysis,
+  isRegressionAnalysis,
+  isClassificationAnalysis,
+  getPredictionFieldName,
+  ANALYSIS_CONFIG_TYPE,
+};
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index b705b585cbacd..b30fd5da2bf0c 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -10,6 +10,7 @@ import { EuiDataGridColumn } from '@elastic/eui';
 
 import { CoreSetup } from 'src/core/public';
 
+import { i18n } from '@kbn/i18n';
 import { MlApiServices } from '../../../../../services/ml_api_service';
 import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
 
@@ -37,6 +38,7 @@ import {
 } from '../../../../common/constants';
 import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields';
 import { isRegressionAnalysis } from '../../../../common/analytics';
+import { extractErrorMessage } from '../../../../../../../common/util/errors';
 
 export const useExplorationResults = (
   indexPattern: IndexPattern | undefined,
@@ -124,18 +126,23 @@ export const useExplorationResults = (
         jobConfig.analysis !== undefined &&
         isRegressionAnalysis(jobConfig.analysis)
       ) {
-        const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(
-          jobConfig.id,
-          jobConfig.dest.index,
-          predictionFieldName
-        );
+        const result = await mlApiServices.dataFrameAnalytics.getAnalyticsBaseline(jobConfig.id);
         if (result?.baseline) {
           setBaseLine(result.baseline);
         }
       }
     } catch (e) {
-      // eslint-disable-next-line
-      console.error(e);
+      const error = extractErrorMessage(e);
+
+      toastNotifications.addDanger({
+        title: i18n.translate(
+          'xpack.ml.dataframe.analytics.explorationResults.baselineErrorMessageToast',
+          {
+            defaultMessage: 'An error occurred getting feature importance baseline',
+          }
+        ),
+        text: error,
+      });
     }
   }, [mlApiServices, jobConfig]);
 
diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
index 2199d54245aec..434200d0383f5 100644
--- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
@@ -135,12 +135,10 @@ export const dataFrameAnalytics = {
       method: 'GET',
     });
   },
-  getAnalyticsBaseline(analyticsId: string, destinationIndex: string, predictionField?: string) {
-    const body = JSON.stringify({ destinationIndex, predictionField });
+  getAnalyticsBaseline(analyticsId: string) {
     return http<any>({
-      path: `${basePath()}/data_frame/analytics/baseline`,
+      path: `${basePath()}/data_frame/analytics/${analyticsId}/baseline`,
       method: 'POST',
-      body,
     });
   },
 };
diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
new file mode 100644
index 0000000000000..9ec5a024a8924
--- /dev/null
+++ b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ILegacyScopedClusterClient } from 'kibana/server';
+import { getPredictionFieldName, isRegressionAnalysis } from '../../../common/util/analytics_utils';
+
+// Obtains data for the data frame analytics feature importance functionalities
+// such as baseline, decision paths, or importance summary.
+export function analyticsFeatureImportanceProvider({
+  callAsCurrentUser,
+  callAsInternalUser,
+}: ILegacyScopedClusterClient) {
+  async function getRegressionAnalyticsBaseline(analyticsId: string): Promise<number | undefined> {
+    const results = await callAsInternalUser('ml.getDataFrameAnalytics', {
+      analyticsId,
+    });
+    const jobConfig = results.data_frame_analytics[0];
+    if (!isRegressionAnalysis) return undefined;
+    const destinationIndex = jobConfig.dest.index;
+    const predictionField = getPredictionFieldName(jobConfig.analysis);
+    const params = {
+      index: destinationIndex,
+      size: 0,
+      body: {
+        query: {
+          bool: {
+            filter: [
+              {
+                term: {
+                  'ml.is_training': true,
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          featureImportanceBaseline: {
+            avg: {
+              field: `ml.${predictionField}`,
+            },
+          },
+        },
+      },
+    };
+    let baseline;
+    const aggregationResult = await callAsCurrentUser('search', params);
+    if (aggregationResult) {
+      baseline = aggregationResult.aggregations.featureImportanceBaseline.value;
+    }
+    return baseline;
+  }
+
+  return {
+    getRegressionAnalyticsBaseline,
+  };
+}
diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
index f02df32234f5a..a75695069c1c7 100644
--- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
@@ -6,7 +6,7 @@
 
 import { RequestHandlerContext, ILegacyScopedClusterClient } from 'kibana/server';
 import { wrapError } from '../client/error_wrapper';
-import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages';
+import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics';
 import { RouteInitialization } from '../types';
 import {
   dataAnalyticsJobConfigSchema,
@@ -21,6 +21,7 @@ import {
 import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns';
 import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics';
 import { getAuthorizationHeader } from '../lib/request_authorization';
+import { analyticsFeatureImportanceProvider } from '../models/data_frame_analytics/feature_importance';
 
 function getIndexPatternId(context: RequestHandlerContext, patternName: string) {
   const iph = new IndexPatternHandler(context.core.savedObjects.client);
@@ -138,7 +139,7 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
         tags: ['access:ml:canGetDataFrameAnalytics'],
       },
     },
-    mlLicense.fullLicenseAPIGuard(async ({ legacyClient, request, response }) => {
+    mlLicense.fullLicenseAPIGuard(async ({ legacyClient, response }) => {
       try {
         const results = await legacyClient.callAsInternalUser('ml.getDataFrameAnalyticsStats');
         return response.ok({
@@ -548,12 +549,13 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
    * @apiName GetDataFrameAnalyticsBaseline
    * @apiDescription Returns the baseline for data frame analytics job.
    *
-   * @apiSchema (params) getDataFrameAnalyticsBaselineSchema
+   * @apiSchema (params) analyticsIdSchema
    */
   router.post(
     {
-      path: '/api/ml/data_frame/analytics/baseline',
+      path: '/api/ml/data_frame/analytics/{analyticsId}/baseline',
       validate: {
+        params: analyticsIdSchema,
         body: getDataFrameAnalyticsBaselineSchema,
       },
       options: {
@@ -562,36 +564,10 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
     },
     mlLicense.fullLicenseAPIGuard(async ({ legacyClient, request, response }) => {
       try {
-        const { destinationIndex, predictionField } = request.body;
-        const params = {
-          index: destinationIndex,
-          size: 0,
-          body: {
-            query: {
-              bool: {
-                filter: [
-                  {
-                    term: {
-                      'ml.is_training': true,
-                    },
-                  },
-                ],
-              },
-            },
-            aggs: {
-              featureImportanceBaseline: {
-                avg: {
-                  field: `ml.${predictionField}`,
-                },
-              },
-            },
-          },
-        };
-        let baseline;
-        const aggregationResult = await legacyClient.callAsCurrentUser('search', params);
-        if (aggregationResult) {
-          baseline = aggregationResult.aggregations.featureImportanceBaseline.value;
-        }
+        const { analyticsId } = request.params;
+        const { getRegressionAnalyticsBaseline } = analyticsFeatureImportanceProvider(legacyClient);
+        const baseline = await getRegressionAnalyticsBaseline(analyticsId);
+
         return response.ok({
           body: { baseline },
         });

From c5b68d5d7dacea5a0799be9a14c1a7c1dc795f7e Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 21 Aug 2020 09:09:31 -0500
Subject: [PATCH 23/37] [ML] Remove getDataFrameAnalyticsBaselineSchema and
 body

---
 x-pack/plugins/ml/server/routes/data_frame_analytics.ts   | 2 --
 .../ml/server/routes/schemas/data_analytics_schema.ts     | 8 --------
 2 files changed, 10 deletions(-)

diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
index a75695069c1c7..24793f5afb242 100644
--- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts
@@ -16,7 +16,6 @@ import {
   analyticsIdSchema,
   stopsDataFrameAnalyticsJobQuerySchema,
   deleteDataFrameAnalyticsJobSchema,
-  getDataFrameAnalyticsBaselineSchema,
 } from './schemas/data_analytics_schema';
 import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns';
 import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics';
@@ -556,7 +555,6 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense }: RouteInitializat
       path: '/api/ml/data_frame/analytics/{analyticsId}/baseline',
       validate: {
         params: analyticsIdSchema,
-        body: getDataFrameAnalyticsBaselineSchema,
       },
       options: {
         tags: ['access:ml:canGetDataFrameAnalytics'],
diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
index a92586ecab9b3..0c3e186c314cc 100644
--- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
+++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
@@ -81,11 +81,3 @@ export const dataAnalyticsJobUpdateSchema = schema.object({
 export const stopsDataFrameAnalyticsJobQuerySchema = schema.object({
   force: schema.maybe(schema.boolean()),
 });
-
-export const getDataFrameAnalyticsBaselineSchema = schema.object({
-  /**
-   * Analytics Baseline
-   */
-  destinationIndex: schema.string(),
-  predictionField: schema.string(),
-});

From 0fa835ff4f907b55d3786c8c34bc76e471555330 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 21 Aug 2020 11:18:03 -0500
Subject: [PATCH 24/37] [ML] Update typo at the predicted value

---
 .../data_grid/feature_importance/decision_path_popover.tsx      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 84991b803b467..2abd451a4296c 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -86,7 +86,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
           <EuiText size={'xs'} color="subdued" style={{ paddingTop: 5 }}>
             <FormattedMessage
               id="xpack.ml.dataframe.analytics.explorationResults.decisionPathPlotHelpText"
-              defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted at the predicted value for '{predictionFieldName}'."
+              defaultMessage="SHAP decision plots use {linkedFeatureImportanceValues} to show how models arrive at the predicted value for '{predictionFieldName}'."
               values={{
                 predictionFieldName,
                 linkedFeatureImportanceValues: (

From 669c68364ac6e137776e8ef056a440a4ddcdcf21 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 21 Aug 2020 11:21:56 -0500
Subject: [PATCH 25/37] [ML] Remove duplicate code in dfa creation

---
 .../ml/data_frame_analytics_creation.ts       | 22 +++----------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts
index ffa1d9fd46c75..e01e065867ac7 100644
--- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts
+++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts
@@ -10,25 +10,9 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 import { MlCommonUI } from './common_ui';
 import { MlApi } from './api';
 import {
-  ClassificationAnalysis,
-  RegressionAnalysis,
-} from '../../../../plugins/ml/common/types/data_frame_analytics';
-
-enum ANALYSIS_CONFIG_TYPE {
-  OUTLIER_DETECTION = 'outlier_detection',
-  REGRESSION = 'regression',
-  CLASSIFICATION = 'classification',
-}
-
-const isRegressionAnalysis = (arg: any): arg is RegressionAnalysis => {
-  const keys = Object.keys(arg);
-  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.REGRESSION;
-};
-
-const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysis => {
-  const keys = Object.keys(arg);
-  return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
-};
+  isRegressionAnalysis,
+  isClassificationAnalysis,
+} from '../../../../plugins/ml/common/util/analytics_utils';
 
 export function MachineLearningDataFrameAnalyticsCreationProvider(
   { getService }: FtrProviderContext,

From eccdb4d141c29c9e29b5293429fafa99965b8f77 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 21 Aug 2020 11:54:34 -0500
Subject: [PATCH 26/37] [ML] Fix datagrid popover pagination crash

---
 .../ml/public/application/components/data_grid/data_grid.tsx   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index a736b0e9ce011..152b3db5e68e8 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -100,8 +100,9 @@ export const DataGrid: FC<Props> = memo(
         analysisType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION
         ? {
             featureImportance: ({ children }: { cellContentsElement: any; children: any }) => {
-              const rowIndex = children?.props?.rowIndex;
+              const rowIndex = children?.props?.visibleRowIndex;
               const row = data[rowIndex];
+              if (!row) return <div />;
               const parsedFIArray = row.ml.feature_importance;
               let predictedValue: string | number | undefined;
               let topClasses: TopClasses = [];

From 049ba5a04aff5bb51b92d23ee6aa62cc3dedeeda Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 21 Aug 2020 12:03:43 -0500
Subject: [PATCH 27/37] [ML] Fix domain calc not working reliably

---
 .../feature_importance/decision_path_classification.tsx     | 6 +++++-
 .../feature_importance/decision_path_regression.tsx         | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
index 051974946a9f2..569e5a3c72d39 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -49,7 +49,11 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
     let maxDomain;
     let minDomain;
     // if decisionPathData has calculated cumulative path
-    if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+    if (
+      Array.isArray(decisionPathData) &&
+      decisionPathData.length > 0 &&
+      decisionPathData[0].length === 3
+    ) {
       const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
       const buffer = Math.abs(max - min) * 0.1;
       maxDomain = max + buffer;
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
index a0b39ee60da37..010aff15de295 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
@@ -34,7 +34,11 @@ export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
     let maxDomain;
     let minDomain;
     // if decisionPathData has calculated cumulative path
-    if (Array.isArray(decisionPathData) && decisionPathData.length === 3) {
+    if (
+      Array.isArray(decisionPathData) &&
+      decisionPathData.length > 0 &&
+      decisionPathData[0].length === 3
+    ) {
       const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
       maxDomain = max;
       minDomain = min;

From ef815d11a2bf334d703366daff3519b38b848e11 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Mon, 24 Aug 2020 12:56:23 -0500
Subject: [PATCH 28/37] [ML]  Minor refactoring - useMemo for decision path
 data - use dest.results_field instead of hardcoded .ml - predictionfieldName
 uses default name if not available - use d3.extent instead of custom max min
 func - return eui callout if decision path data for some reason is not
 available - new isDecisionPathData() - tickFormatter doesn't need template
 literal

---
 .../plugins/ml/common/util/analytics_utils.ts | 33 +++++++++++++++++
 .../components/data_grid/data_grid.tsx        | 12 ++++---
 .../decision_path_chart.tsx                   |  4 +--
 .../decision_path_classification.tsx          | 15 ++++----
 .../decision_path_regression.tsx              | 14 ++++----
 .../missing_decision_path_callout.tsx         | 20 +++++++++++
 .../use_classification_path_data.tsx          | 31 ++++++----------
 .../application/components/data_grid/types.ts |  2 ++
 .../data_frame_analytics/common/analytics.ts  | 35 +++----------------
 .../data_frame_analytics/common/fields.ts     |  7 ++--
 .../use_exploration_results.ts                | 21 +++++++----
 11 files changed, 110 insertions(+), 84 deletions(-)
 create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx

diff --git a/x-pack/plugins/ml/common/util/analytics_utils.ts b/x-pack/plugins/ml/common/util/analytics_utils.ts
index 6cafbafc528af..d725984a47d66 100644
--- a/x-pack/plugins/ml/common/util/analytics_utils.ts
+++ b/x-pack/plugins/ml/common/util/analytics_utils.ts
@@ -27,6 +27,23 @@ export const isClassificationAnalysis = (arg: any): arg is ClassificationAnalysi
   return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
 };
 
+export const getDependentVar = (
+  analysis: AnalysisConfig
+):
+  | RegressionAnalysis['regression']['dependent_variable']
+  | ClassificationAnalysis['classification']['dependent_variable'] => {
+  let depVar = '';
+
+  if (isRegressionAnalysis(analysis)) {
+    depVar = analysis.regression.dependent_variable;
+  }
+
+  if (isClassificationAnalysis(analysis)) {
+    depVar = analysis.classification.dependent_variable;
+  }
+  return depVar;
+};
+
 export const getPredictionFieldName = (
   analysis: AnalysisConfig
 ):
@@ -44,3 +61,19 @@ export const getPredictionFieldName = (
   }
   return predictionFieldName;
 };
+
+export const getDefaultPredictionFieldName = (analysis: AnalysisConfig) => {
+  return `${getDependentVar(analysis)}_prediction`;
+};
+export const getPredictedFieldName = (
+  resultsField: string,
+  analysis: AnalysisConfig,
+  forSort?: boolean
+) => {
+  // default is 'ml'
+  const predictionFieldName = getPredictionFieldName(analysis);
+  const predictedField = `${resultsField}.${
+    predictionFieldName ? predictionFieldName : getDefaultPredictionFieldName(analysis)
+  }`;
+  return predictedField;
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 152b3db5e68e8..63e08d2c34d2e 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -44,6 +44,7 @@ export const DataGridTitle: FC<{ title: string }> = ({ title }) => (
 interface PropsWithoutHeader extends UseIndexDataReturnType {
   baseline?: number;
   analysisType?: ANALYSIS_CONFIG_TYPE;
+  resultsField?: string;
   dataTestSubj: string;
   toastNotifications: CoreSetup['notifications']['toasts'];
 }
@@ -85,6 +86,7 @@ export const DataGrid: FC<Props> = memo(
       toggleChartVisibility,
       visibleColumns,
       predictionFieldName,
+      resultsField,
       analysisType,
     } = props;
     // TODO Fix row hovering + bar highlighting
@@ -103,16 +105,18 @@ export const DataGrid: FC<Props> = memo(
               const rowIndex = children?.props?.visibleRowIndex;
               const row = data[rowIndex];
               if (!row) return <div />;
-              const parsedFIArray = row.ml.feature_importance;
+              // if resultsField for some reason is not available then use ml
+              const mlResultsField = resultsField ?? 'ml';
+              const parsedFIArray = row[mlResultsField].feature_importance;
               let predictedValue: string | number | undefined;
               let topClasses: TopClasses = [];
               if (
                 predictionFieldName !== undefined &&
                 row &&
-                row.ml[predictionFieldName] !== undefined
+                row[mlResultsField][predictionFieldName] !== undefined
               ) {
-                predictedValue = row.ml[predictionFieldName];
-                topClasses = row.ml.top_classes;
+                predictedValue = row[mlResultsField][predictionFieldName];
+                topClasses = row[mlResultsField].top_classes;
               }
 
               return (
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 2dbfccfffdcd5..b6a3810752522 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-// adjust the height so it's compact for items with more features
 import {
   AnnotationDomainTypes,
   Axis,
@@ -64,6 +63,7 @@ export const DecisionPathChart = ({
   maxDomain,
   baseline,
 }: DecisionPathChartProps) => {
+  // adjust the height so it's compact for items with more features
   const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 4 ? 30 : 75;
   const baselineData: LineAnnotationDatum[] = useMemo(
     () => [
@@ -80,7 +80,7 @@ export const DecisionPathChart = ({
     ],
     [baseline]
   );
-  const tickFormatter = useCallback((d) => `${Number(d).toPrecision(3)}`, []);
+  const tickFormatter = useCallback((d) => Number(d).toPrecision(3), []);
 
   return (
     <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
index 569e5a3c72d39..8703c7c49bc06 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -7,9 +7,12 @@
 import React, { FC, useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiHealth, EuiSpacer, EuiSuperSelect, EuiTitle } from '@elastic/eui';
-import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
+import d3 from 'd3';
+import { isDecisionPathData, useDecisionPathData } from './use_classification_path_data';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { DecisionPathChart } from './decision_path_chart';
+import { MissingDecisionPathCallout } from './missing_decision_path_callout';
+
 interface ClassificationDecisionPathProps {
   predictedValue: string | undefined;
   predictionFieldName?: string;
@@ -49,12 +52,8 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
     let maxDomain;
     let minDomain;
     // if decisionPathData has calculated cumulative path
-    if (
-      Array.isArray(decisionPathData) &&
-      decisionPathData.length > 0 &&
-      decisionPathData[0].length === 3
-    ) {
-      const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
+    if (decisionPathData && isDecisionPathData(decisionPathData)) {
+      const [min, max] = d3.extent(decisionPathData, (d: [string, number, number]) => d[2]);
       const buffer = Math.abs(max - min) * 0.1;
       maxDomain = max + buffer;
       minDomain = min - buffer;
@@ -62,7 +61,7 @@ export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = (
     return { maxDomain, minDomain };
   }, [decisionPathData]);
 
-  if (!decisionPathData) return <div />;
+  if (!decisionPathData) return <MissingDecisionPathCallout />;
 
   return (
     <>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
index 010aff15de295..345269a944f02 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_regression.tsx
@@ -7,9 +7,11 @@
 import React, { FC, useMemo } from 'react';
 import { EuiCallOut } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
+import d3 from 'd3';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
-import { findMaxMin, useDecisionPathData } from './use_classification_path_data';
+import { useDecisionPathData, isDecisionPathData } from './use_classification_path_data';
 import { DecisionPathChart } from './decision_path_chart';
+import { MissingDecisionPathCallout } from './missing_decision_path_callout';
 
 interface RegressionDecisionPathProps {
   predictionFieldName?: string;
@@ -34,12 +36,8 @@ export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
     let maxDomain;
     let minDomain;
     // if decisionPathData has calculated cumulative path
-    if (
-      Array.isArray(decisionPathData) &&
-      decisionPathData.length > 0 &&
-      decisionPathData[0].length === 3
-    ) {
-      const { max, min } = findMaxMin(decisionPathData, (d: [string, number, number]) => d[2]);
+    if (decisionPathData && isDecisionPathData(decisionPathData)) {
+      const [min, max] = d3.extent(decisionPathData, (d: [string, number, number]) => d[2]);
       maxDomain = max;
       minDomain = min;
       const buffer = Math.abs(maxDomain - minDomain) * 0.1;
@@ -51,7 +49,7 @@ export const RegressionDecisionPath: FC<RegressionDecisionPathProps> = ({
     return { maxDomain, minDomain };
   }, [decisionPathData, baseline]);
 
-  if (!decisionPathData) return <div />;
+  if (!decisionPathData) return <MissingDecisionPathCallout />;
 
   return (
     <>
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx
new file mode 100644
index 0000000000000..66eb2047b1314
--- /dev/null
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/missing_decision_path_callout.tsx
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiCallOut } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+export const MissingDecisionPathCallout = () => {
+  return (
+    <EuiCallOut color={'warning'}>
+      <FormattedMessage
+        id="xpack.ml.dataframe.analytics.explorationResults.regressionDecisionPathDataMissingCallout"
+        defaultMessage="No decision path data available."
+      />
+    </EuiCallOut>
+  );
+};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
index b06516e4d868e..a86c3b9c8603d 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useEffect, useState } from 'react';
+import { useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { ExtendedFeatureImportance } from './decision_path_popover';
@@ -27,15 +27,20 @@ interface RegressionDecisionPathProps {
 const FEATURE_NAME = 'feature_name';
 const FEATURE_IMPORTANCE = 'importance';
 
+export const isDecisionPathData = (decisionPathData: any): boolean => {
+  return (
+    Array.isArray(decisionPathData) &&
+    decisionPathData.length > 0 &&
+    decisionPathData[0].length === 3
+  );
+};
 export const useDecisionPathData = ({
   baseline,
   featureImportance,
   predictedValue,
 }: UseDecisionPathDataParams): { decisionPathData: DecisionPathPlotData | undefined } => {
-  const [decisionPathData, setDecisionPlotData] = useState<DecisionPathPlotData | undefined>();
-
-  useEffect(() => {
-    const result = baseline
+  const decisionPathData = useMemo(() => {
+    return baseline
       ? buildRegressionDecisionPathData({
           baseline,
           featureImportance,
@@ -45,8 +50,6 @@ export const useDecisionPathData = ({
           featureImportance,
           currentClass: predictedValue as string | undefined,
         });
-
-    setDecisionPlotData(result);
   }, [baseline, featureImportance, predictedValue]);
 
   return { decisionPathData };
@@ -153,17 +156,3 @@ export const buildClassificationDecisionPathData = ({
 
   return buildDecisionPathData(filteredFeatureImportance);
 };
-
-export const findMaxMin = (
-  data: DecisionPathPlotData,
-  getter: Function
-): { max: number; min: number } => {
-  let min = Infinity;
-  let max = -Infinity;
-  data.forEach((d) => {
-    const value = getter(d);
-    if (value > max) max = value;
-    if (value < min) min = value;
-  });
-  return { max, min };
-};
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/types.ts b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
index c2cf6d28c2794..f9ee8c37fabf7 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/types.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/types.ts
@@ -76,6 +76,7 @@ export interface UseIndexDataReturnType
     | 'visibleColumns'
     | 'baseline'
     | 'predictionFieldName'
+    | 'resultsField'
   > {
   renderCellValue: RenderCellValue;
 }
@@ -109,4 +110,5 @@ export interface UseDataGridReturnType {
   visibleColumns: ColumnId[];
   baseline?: number;
   predictionFieldName?: string;
+  resultsField?: string;
 }
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
index e6b9197da9099..97098ea9e75c6 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts
@@ -23,6 +23,8 @@ import {
   isRegressionAnalysis,
   isClassificationAnalysis,
   getPredictionFieldName,
+  getDependentVar,
+  getPredictedFieldName,
 } from '../../../../common/util/analytics_utils';
 export type IndexPattern = string;
 
@@ -155,23 +157,6 @@ export const getAnalysisType = (analysis: AnalysisConfig): string => {
   return 'unknown';
 };
 
-export const getDependentVar = (
-  analysis: AnalysisConfig
-):
-  | RegressionAnalysis['regression']['dependent_variable']
-  | ClassificationAnalysis['classification']['dependent_variable'] => {
-  let depVar = '';
-
-  if (isRegressionAnalysis(analysis)) {
-    depVar = analysis.regression.dependent_variable;
-  }
-
-  if (isClassificationAnalysis(analysis)) {
-    depVar = analysis.classification.dependent_variable;
-  }
-  return depVar;
-};
-
 export const getTrainingPercent = (
   analysis: AnalysisConfig
 ):
@@ -219,20 +204,6 @@ export const getNumTopFeatureImportanceValues = (
   return numTopFeatureImportanceValues;
 };
 
-export const getPredictedFieldName = (
-  resultsField: string,
-  analysis: AnalysisConfig,
-  forSort?: boolean
-) => {
-  // default is 'ml'
-  const predictionFieldName = getPredictionFieldName(analysis);
-  const defaultPredictionField = `${getDependentVar(analysis)}_prediction`;
-  const predictedField = `${resultsField}.${
-    predictionFieldName ? predictionFieldName : defaultPredictionField
-  }`;
-  return predictedField;
-};
-
 export const isResultsSearchBoolQuery = (arg: any): arg is ResultsSearchBoolQuery => {
   if (arg === undefined) return false;
   const keys = Object.keys(arg);
@@ -580,4 +551,6 @@ export {
   isClassificationAnalysis,
   getPredictionFieldName,
   ANALYSIS_CONFIG_TYPE,
+  getDependentVar,
+  getPredictedFieldName,
 };
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
index 847aefefbc6c8..f9c9bf26a9d16 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
@@ -4,17 +4,16 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { getNumTopClasses, getNumTopFeatureImportanceValues } from './analytics';
+import { Field } from '../../../../common/types/fields';
 import {
-  getNumTopClasses,
-  getNumTopFeatureImportanceValues,
   getPredictedFieldName,
   getDependentVar,
   getPredictionFieldName,
   isClassificationAnalysis,
   isOutlierAnalysis,
   isRegressionAnalysis,
-} from './analytics';
-import { Field } from '../../../../common/types/fields';
+} from '../../../../common/util/analytics_utils';
 import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 import { newJobCapsService } from '../../services/new_job_capabilities_service';
 
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index b30fd5da2bf0c..13826284e5242 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -25,12 +25,11 @@ import {
   UseIndexDataReturnType,
 } from '../../../../../components/data_grid';
 import { SavedSearchQuery } from '../../../../../contexts/ml';
+import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common';
 import {
-  getIndexData,
-  getIndexFields,
-  DataFrameAnalyticsConfig,
   getPredictionFieldName,
-} from '../../../../common';
+  getDefaultPredictionFieldName,
+} from '../../../../../../../common/util/analytics_utils';
 import {
   DEFAULT_RESULTS_FIELD,
   FEATURE_IMPORTANCE,
@@ -117,7 +116,15 @@ export const useExplorationResults = (
     jobConfig?.dest.index,
     JSON.stringify([searchQuery, dataGrid.visibleColumns]),
   ]);
-  const predictionFieldName = jobConfig ? getPredictionFieldName(jobConfig.analysis) : undefined;
+  const predictionFieldName = useMemo(() => {
+    if (jobConfig) {
+      return (
+        getPredictionFieldName(jobConfig.analysis) ??
+        getDefaultPredictionFieldName(jobConfig.analysis)
+      );
+    }
+    return undefined;
+  }, [jobConfig]);
 
   const getAnalyticsBaseline = useCallback(async () => {
     try {
@@ -150,11 +157,12 @@ export const useExplorationResults = (
     getAnalyticsBaseline();
   }, [jobConfig]);
 
+  const resultsField = jobConfig?.dest.results_field ?? DEFAULT_RESULTS_FIELD;
   const renderCellValue = useRenderCellValue(
     indexPattern,
     dataGrid.pagination,
     dataGrid.tableItems,
-    jobConfig?.dest.results_field ?? DEFAULT_RESULTS_FIELD
+    resultsField
   );
 
   return {
@@ -162,5 +170,6 @@ export const useExplorationResults = (
     renderCellValue,
     baseline,
     predictionFieldName,
+    resultsField,
   };
 };

From 74ef5fd02e9143582cfcdf080388e5d93b749c9f Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Mon, 24 Aug 2020 13:45:12 -0500
Subject: [PATCH 29/37] [ML]  Fix so classification support boolean class names

---
 .../ml/common/types/feature_importance.ts     |  2 +-
 .../decision_path_classification.tsx          | 36 +++++++++++--------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugins/ml/common/types/feature_importance.ts b/x-pack/plugins/ml/common/types/feature_importance.ts
index b44ebcb51c199..d2ab9f6c58608 100644
--- a/x-pack/plugins/ml/common/types/feature_importance.ts
+++ b/x-pack/plugins/ml/common/types/feature_importance.ts
@@ -5,7 +5,7 @@
  */
 
 export interface ClassFeatureImportance {
-  class_name: string;
+  class_name: string | boolean;
   importance: number;
 }
 export interface FeatureImportance {
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
index 8703c7c49bc06..5480b21d6eb96 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -14,40 +14,48 @@ import { DecisionPathChart } from './decision_path_chart';
 import { MissingDecisionPathCallout } from './missing_decision_path_callout';
 
 interface ClassificationDecisionPathProps {
-  predictedValue: string | undefined;
+  predictedValue: string | boolean;
   predictionFieldName?: string;
   featureImportance: FeatureImportance[];
   topClasses: TopClasses;
 }
 
+// cast to 'True' | 'False' | value to match Eui display
+const getStr = (v: string | boolean): string =>
+  typeof v === 'boolean' ? (v ? 'True' : 'False') : v;
+
 export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = ({
   featureImportance,
   predictedValue,
   topClasses,
   predictionFieldName,
 }) => {
-  const [currentClass, setCurrentClass] = useState<string>(topClasses[0].class_name);
+  const [currentClass, setCurrentClass] = useState<string>(getStr(topClasses[0].class_name));
   const { decisionPathData } = useDecisionPathData({
     featureImportance,
     predictedValue: currentClass,
   });
-  const options = useMemo(
-    () =>
-      Array.isArray(topClasses) && typeof predictedValue === 'string'
-        ? topClasses.map((c) => ({
-            value: c.class_name,
+  const options = useMemo(() => {
+    const predictionValueStr = getStr(predictedValue);
+
+    return Array.isArray(topClasses)
+      ? topClasses.map((c) => {
+          const className = getStr(c.class_name);
+          return {
+            value: className,
             inputDisplay:
-              c.class_name === predictedValue ? (
+              className === predictionValueStr ? (
                 <EuiHealth color="success" style={{ lineHeight: 'inherit' }}>
-                  {c.class_name}
+                  {className}
                 </EuiHealth>
               ) : (
-                c.class_name
+                className
               ),
-          }))
-        : undefined,
-    [topClasses, predictedValue]
-  );
+          };
+        })
+      : undefined;
+  }, [topClasses, predictedValue]);
+
   const domain = useMemo(() => {
     let maxDomain;
     let minDomain;

From eaae596da7f49f619321662fb2791c7949dfac4b Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Mon, 24 Aug 2020 15:09:26 -0500
Subject: [PATCH 30/37] [ML]  Fix better height and change to empty annotation
 marker

---
 .../feature_importance/decision_path_chart.tsx     | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index b6a3810752522..0dc3b4584ab63 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -56,6 +56,11 @@ interface DecisionPathChartProps {
   maxDomain: number | undefined;
 }
 
+const DECISION_PATH_MARGIN = 125;
+const DECISION_PATH_ROW_HEIGHT = 10;
+
+const AnnotationBaselineMarker = <EuiIcon type={'empty'} size={'s'} />;
+
 export const DecisionPathChart = ({
   decisionPathData,
   predictionFieldName,
@@ -64,7 +69,6 @@ export const DecisionPathChart = ({
   baseline,
 }: DecisionPathChartProps) => {
   // adjust the height so it's compact for items with more features
-  const heightMultiplier = Array.isArray(decisionPathData) && decisionPathData.length > 4 ? 30 : 75;
   const baselineData: LineAnnotationDatum[] = useMemo(
     () => [
       {
@@ -83,7 +87,9 @@ export const DecisionPathChart = ({
   const tickFormatter = useCallback((d) => Number(d).toPrecision(3), []);
 
   return (
-    <Chart size={{ height: decisionPathData.length * heightMultiplier }}>
+    <Chart
+      size={{ height: DECISION_PATH_MARGIN + decisionPathData.length * DECISION_PATH_ROW_HEIGHT }}
+    >
       <Settings theme={theme} rotation={90} />
       {baseline && (
         <LineAnnotation
@@ -91,7 +97,7 @@ export const DecisionPathChart = ({
           domainType={AnnotationDomainTypes.YDomain}
           dataValues={baselineData}
           style={baselineStyle}
-          marker={<EuiIcon type={'annotation'} />}
+          marker={AnnotationBaselineMarker}
         />
       )}
 
@@ -106,7 +112,7 @@ export const DecisionPathChart = ({
           }
         )}
         showGridLines={true}
-        position={Position.Bottom}
+        position={Position.Top}
         showOverlappingTicks
         domain={
           minDomain && maxDomain

From 9aa5d818f811def50962940b91d320a574af245a Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 25 Aug 2020 17:14:03 -0500
Subject: [PATCH 31/37] [ML] Adjust style & precision header for baseline

---
 .../decision_path_chart.tsx                   | 41 ++++++++++++++-----
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 0dc3b4584ab63..4f142b6971d9e 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -10,6 +10,7 @@ import {
   AxisConfig,
   Chart,
   LineAnnotation,
+  LineAnnotationStyle,
   LineAnnotationDatum,
   LineSeries,
   PartialTheme,
@@ -22,26 +23,45 @@ import { EuiIcon } from '@elastic/eui';
 
 import React, { useCallback, useMemo } from 'react';
 import { i18n } from '@kbn/i18n';
+import euiVars from '@elastic/eui/dist/eui_theme_light.json';
 import { DecisionPathPlotData } from './use_classification_path_data';
 
-const baselineStyle = {
+const { euiColorFullShade, euiColorMediumShade } = euiVars;
+const axisColor = euiColorMediumShade;
+
+const baselineStyle: LineAnnotationStyle = {
   line: {
     strokeWidth: 1,
-    stroke: 'gray',
-    opacity: 1,
+    stroke: euiColorFullShade,
+    opacity: 0.75,
   },
   details: {
-    fontSize: 12,
     fontFamily: 'Arial',
+    fontSize: 10,
     fontStyle: 'bold',
-    fill: 'gray',
+    fill: euiColorMediumShade,
     padding: 0,
   },
 };
 
 const axes: RecursivePartial<AxisConfig> = {
+  axisLineStyle: {
+    stroke: axisColor,
+  },
   tickLabelStyle: {
-    fontSize: 12,
+    fontSize: 10,
+    fill: axisColor,
+  },
+  tickLineStyle: {
+    stroke: axisColor,
+  },
+  gridLineStyle: {
+    horizontal: {
+      dash: [1, 2],
+    },
+    vertical: {
+      strokeWidth: 0,
+    },
   },
 };
 const theme: PartialTheme = {
@@ -58,8 +78,8 @@ interface DecisionPathChartProps {
 
 const DECISION_PATH_MARGIN = 125;
 const DECISION_PATH_ROW_HEIGHT = 10;
-
-const AnnotationBaselineMarker = <EuiIcon type={'empty'} size={'s'} />;
+const NUM_PRECISION = 3;
+const AnnotationBaselineMarker = <EuiIcon type="dot" size="m" />;
 
 export const DecisionPathChart = ({
   decisionPathData,
@@ -72,7 +92,7 @@ export const DecisionPathChart = ({
   const baselineData: LineAnnotationDatum[] = useMemo(
     () => [
       {
-        dataValue: baseline ? parseFloat(baseline.toPrecision(3)) : undefined,
+        dataValue: baseline ? baseline : undefined,
         details: i18n.translate(
           'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText',
           {
@@ -84,7 +104,7 @@ export const DecisionPathChart = ({
     ],
     [baseline]
   );
-  const tickFormatter = useCallback((d) => Number(d).toPrecision(3), []);
+  const tickFormatter = useCallback((d) => Number(d).toPrecision(NUM_PRECISION), []);
 
   return (
     <Chart
@@ -98,6 +118,7 @@ export const DecisionPathChart = ({
           dataValues={baselineData}
           style={baselineStyle}
           marker={AnnotationBaselineMarker}
+          header={baseline.toPrecision(NUM_PRECISION)}
         />
       )}
 

From a71e190ca0f7d9d7edca23ed2b110e5eb6945a7f Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 25 Aug 2020 22:48:51 -0500
Subject: [PATCH 32/37] [ML] Update grid and header

---
 .../data_grid/feature_importance/decision_path_chart.tsx    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index 4f142b6971d9e..a4b17f45bc942 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -92,7 +92,8 @@ export const DecisionPathChart = ({
   const baselineData: LineAnnotationDatum[] = useMemo(
     () => [
       {
-        dataValue: baseline ? baseline : undefined,
+        dataValue: baseline,
+        header: baseline ? baseline.toPrecision(NUM_PRECISION) : '',
         details: i18n.translate(
           'xpack.ml.dataframe.analytics.explorationResults.decisionPathBaselineText',
           {
@@ -118,7 +119,6 @@ export const DecisionPathChart = ({
           dataValues={baselineData}
           style={baselineStyle}
           marker={AnnotationBaselineMarker}
-          header={baseline.toPrecision(NUM_PRECISION)}
         />
       )}
 
@@ -132,7 +132,7 @@ export const DecisionPathChart = ({
             values: { predictionFieldName },
           }
         )}
-        showGridLines={true}
+        showGridLines={false}
         position={Position.Top}
         showOverlappingTicks
         domain={

From a954cc633e4d414d7ea61ec6a6d9e36dc97d4737 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 3 Sep 2020 09:10:52 -0500
Subject: [PATCH 33/37] [ML] Update to new client & more strict conversion

---
 .../decision_path_classification.tsx           | 18 ++++++++++--------
 .../use_classification_path_data.tsx           | 17 ++++++++++++++++-
 .../data_frame_analytics/feature_importance.ts | 16 ++++++++--------
 3 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
index 5480b21d6eb96..bd001fa81a582 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_classification.tsx
@@ -8,7 +8,11 @@ import React, { FC, useMemo, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiHealth, EuiSpacer, EuiSuperSelect, EuiTitle } from '@elastic/eui';
 import d3 from 'd3';
-import { isDecisionPathData, useDecisionPathData } from './use_classification_path_data';
+import {
+  isDecisionPathData,
+  useDecisionPathData,
+  getStringBasedClassName,
+} from './use_classification_path_data';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { DecisionPathChart } from './decision_path_chart';
 import { MissingDecisionPathCallout } from './missing_decision_path_callout';
@@ -20,27 +24,25 @@ interface ClassificationDecisionPathProps {
   topClasses: TopClasses;
 }
 
-// cast to 'True' | 'False' | value to match Eui display
-const getStr = (v: string | boolean): string =>
-  typeof v === 'boolean' ? (v ? 'True' : 'False') : v;
-
 export const ClassificationDecisionPath: FC<ClassificationDecisionPathProps> = ({
   featureImportance,
   predictedValue,
   topClasses,
   predictionFieldName,
 }) => {
-  const [currentClass, setCurrentClass] = useState<string>(getStr(topClasses[0].class_name));
+  const [currentClass, setCurrentClass] = useState<string>(
+    getStringBasedClassName(topClasses[0].class_name)
+  );
   const { decisionPathData } = useDecisionPathData({
     featureImportance,
     predictedValue: currentClass,
   });
   const options = useMemo(() => {
-    const predictionValueStr = getStr(predictedValue);
+    const predictionValueStr = getStringBasedClassName(predictedValue);
 
     return Array.isArray(topClasses)
       ? topClasses.map((c) => {
-          const className = getStr(c.class_name);
+          const className = getStringBasedClassName(c.class_name);
           return {
             value: className,
             inputDisplay:
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
index a86c3b9c8603d..6fc25570a016c 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -34,6 +34,21 @@ export const isDecisionPathData = (decisionPathData: any): boolean => {
     decisionPathData[0].length === 3
   );
 };
+
+// cast to 'True' | 'False' | value to match Eui display
+export const getStringBasedClassName = (v: string | boolean | undefined | number): string => {
+  if (v === undefined) {
+    return '';
+  }
+  if (typeof v === 'boolean') {
+    return v ? 'True' : 'False';
+  }
+  if (typeof v === 'number') {
+    return v.toString();
+  }
+  return v;
+};
+
 export const useDecisionPathData = ({
   baseline,
   featureImportance,
@@ -139,7 +154,7 @@ export const buildClassificationDecisionPathData = ({
     ExtendedFeatureImportance | undefined
   > = featureImportance.map((feature) => {
     const classFeatureImportance = Array.isArray(feature.classes)
-      ? feature.classes.find((c) => c.class_name === currentClass)
+      ? feature.classes.find((c) => getStringBasedClassName(c.class_name) === currentClass)
       : feature;
     if (classFeatureImportance && typeof classFeatureImportance[FEATURE_IMPORTANCE] === 'number') {
       return {
diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
index 9ec5a024a8924..9c6a47d53ac05 100644
--- a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
+++ b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
@@ -4,20 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ILegacyScopedClusterClient } from 'kibana/server';
+import { IScopedClusterClient } from 'kibana/server';
 import { getPredictionFieldName, isRegressionAnalysis } from '../../../common/util/analytics_utils';
 
 // Obtains data for the data frame analytics feature importance functionalities
 // such as baseline, decision paths, or importance summary.
 export function analyticsFeatureImportanceProvider({
-  callAsCurrentUser,
-  callAsInternalUser,
-}: ILegacyScopedClusterClient) {
+  asInternalUser,
+  asCurrentUser,
+}: IScopedClusterClient) {
   async function getRegressionAnalyticsBaseline(analyticsId: string): Promise<number | undefined> {
-    const results = await callAsInternalUser('ml.getDataFrameAnalytics', {
-      analyticsId,
+    const { body } = await asInternalUser.ml.getDataFrameAnalytics({
+      id: analyticsId,
     });
-    const jobConfig = results.data_frame_analytics[0];
+    const jobConfig = body.data_frame_analytics[0];
     if (!isRegressionAnalysis) return undefined;
     const destinationIndex = jobConfig.dest.index;
     const predictionField = getPredictionFieldName(jobConfig.analysis);
@@ -46,7 +46,7 @@ export function analyticsFeatureImportanceProvider({
       },
     };
     let baseline;
-    const aggregationResult = await callAsCurrentUser('search', params);
+    const { body: aggregationResult } = await asCurrentUser.search(params);
     if (aggregationResult) {
       baseline = aggregationResult.aggregations.featureImportanceBaseline.value;
     }

From 09495e3415c2bbd665570b147f6be93f241fdc6f Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Thu, 3 Sep 2020 10:16:20 -0500
Subject: [PATCH 34/37] [ML] Fix type

---
 .../feature_importance/decision_path_chart.tsx       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index a4b17f45bc942..b5330371c2841 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -7,7 +7,7 @@
 import {
   AnnotationDomainTypes,
   Axis,
-  AxisConfig,
+  AxisStyle,
   Chart,
   LineAnnotation,
   LineAnnotationStyle,
@@ -44,18 +44,18 @@ const baselineStyle: LineAnnotationStyle = {
   },
 };
 
-const axes: RecursivePartial<AxisConfig> = {
-  axisLineStyle: {
+const axes: RecursivePartial<AxisStyle> = {
+  axisLine: {
     stroke: axisColor,
   },
-  tickLabelStyle: {
+  tickLabel: {
     fontSize: 10,
     fill: axisColor,
   },
-  tickLineStyle: {
+  tickLine: {
     stroke: axisColor,
   },
-  gridLineStyle: {
+  gridLine: {
     horizontal: {
       dash: [1, 2],
     },

From 1345cf2de80b3987dc5d25d40d2bdb4740075dc4 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Fri, 4 Sep 2020 10:20:14 -0500
Subject: [PATCH 35/37] [ML] Update tolerance to check for abs difference

---
 .../feature_importance/use_classification_path_data.tsx         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
index 6fc25570a016c..90216c4a58ffc 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/use_classification_path_data.tsx
@@ -122,7 +122,7 @@ export const buildRegressionDecisionPathData = ({
     });
 
     // if the difference is small enough then no need to plot the residual feature importance
-    if (adjustedImportance > 1e-5) {
+    if (Math.abs(adjustedImportance) > 1e-5) {
       mappedFeatureImportance.push({
         [FEATURE_NAME]: i18n.translate(
           'xpack.ml.dataframe.analytics.decisionPathFeatureOtherTitle',

From a93e740ec161bf79b642b68e9dfadb043720c8c2 Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 8 Sep 2020 07:57:32 -0500
Subject: [PATCH 36/37] [ML] Change to DEFAULT_RESULTS_FIELD

---
 .../ml/public/application/components/data_grid/data_grid.tsx   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 63e08d2c34d2e..2fc148a790988 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -31,6 +31,7 @@ import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
 import { UseIndexDataReturnType } from './types';
 import { DecisionPathPopover } from './feature_importance/decision_path_popover';
 import { TopClasses } from '../../../../common/types/feature_importance';
+import { DEFAULT_RESULTS_FIELD } from '../../data_frame_analytics/common/constants';
 
 // TODO Fix row hovering + bar highlighting
 // import { hoveredRow$ } from './column_chart';
@@ -106,7 +107,7 @@ export const DataGrid: FC<Props> = memo(
               const row = data[rowIndex];
               if (!row) return <div />;
               // if resultsField for some reason is not available then use ml
-              const mlResultsField = resultsField ?? 'ml';
+              const mlResultsField = resultsField ?? DEFAULT_RESULTS_FIELD;
               const parsedFIArray = row[mlResultsField].feature_importance;
               let predictedValue: string | number | undefined;
               let topClasses: TopClasses = [];

From 5fd5ff42a0b43d6eef2bce3d3fa3b71683db695e Mon Sep 17 00:00:00 2001
From: Quynh Nguyen <quynh.nguyen@elastic.co>
Date: Tue, 8 Sep 2020 09:16:43 -0500
Subject: [PATCH 37/37] [ML] Update doc link, tick formatter, and baseline api
 for generic results fields

---
 .../common/constants/data_frame_analytics.ts  |  7 +++++++
 .../components/data_grid/data_grid.tsx        |  2 +-
 .../decision_path_chart.tsx                   |  4 +++-
 .../decision_path_popover.tsx                 |  7 ++++++-
 .../data_frame_analytics/common/constants.ts  |  2 --
 .../use_exploration_results.ts                |  7 ++-----
 .../outlier_exploration/use_outlier_data.ts   |  3 ++-
 .../action_clone/clone_action_name.tsx        |  2 +-
 .../feature_importance.ts                     | 20 ++++++++++++++-----
 9 files changed, 37 insertions(+), 17 deletions(-)
 create mode 100644 x-pack/plugins/ml/common/constants/data_frame_analytics.ts

diff --git a/x-pack/plugins/ml/common/constants/data_frame_analytics.ts b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts
new file mode 100644
index 0000000000000..830537cbadbc8
--- /dev/null
+++ b/x-pack/plugins/ml/common/constants/data_frame_analytics.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const DEFAULT_RESULTS_FIELD = 'ml';
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
index 2fc148a790988..22815fe593d57 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx
@@ -31,7 +31,7 @@ import { euiDataGridStyle, euiDataGridToolbarSettings } from './common';
 import { UseIndexDataReturnType } from './types';
 import { DecisionPathPopover } from './feature_importance/decision_path_popover';
 import { TopClasses } from '../../../../common/types/feature_importance';
-import { DEFAULT_RESULTS_FIELD } from '../../data_frame_analytics/common/constants';
+import { DEFAULT_RESULTS_FIELD } from '../../../../common/constants/data_frame_analytics';
 
 // TODO Fix row hovering + bar highlighting
 // import { hoveredRow$ } from './column_chart';
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
index b5330371c2841..b546ac1db57dd 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_chart.tsx
@@ -105,7 +105,9 @@ export const DecisionPathChart = ({
     ],
     [baseline]
   );
-  const tickFormatter = useCallback((d) => Number(d).toPrecision(NUM_PRECISION), []);
+  // guarantee up to num_precision significant digits
+  // without having it in scientific notation
+  const tickFormatter = useCallback((d) => Number(d.toPrecision(NUM_PRECISION)).toString(), []);
 
   return (
     <Chart
diff --git a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
index 2abd451a4296c..263337f93e9a8 100644
--- a/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
+++ b/x-pack/plugins/ml/public/application/components/data_grid/feature_importance/decision_path_popover.tsx
@@ -12,6 +12,7 @@ import { DecisionPathJSONViewer } from './decision_path_json_viewer';
 import { FeatureImportance, TopClasses } from '../../../../../common/types/feature_importance';
 import { ANALYSIS_CONFIG_TYPE } from '../../../data_frame_analytics/common';
 import { ClassificationDecisionPath } from './decision_path_classification';
+import { useMlKibana } from '../../../contexts/kibana';
 
 interface DecisionPathPopoverProps {
   featureImportance: FeatureImportance[];
@@ -40,6 +41,10 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
   predictionFieldName,
 }) => {
   const [selectedTabId, setSelectedTabId] = useState(DECISION_PATH_TABS.CHART);
+  const {
+    services: { docLinks },
+  } = useMlKibana();
+  const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = docLinks;
 
   if (featureImportance.length < 2) {
     return <DecisionPathJSONViewer featureImportance={featureImportance} />;
@@ -91,7 +96,7 @@ export const DecisionPathPopover: FC<DecisionPathPopoverProps> = ({
                 predictionFieldName,
                 linkedFeatureImportanceValues: (
                   <EuiLink
-                    href="https://www.elastic.co/guide/en/machine-learning/7.8/ml-feature-importance.html"
+                    href={`${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-feature-importance.html`}
                     target="_blank"
                   >
                     <FormattedMessage
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts
index 2f14dfdfdfca3..c2295a92af89c 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/constants.ts
@@ -3,8 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
-export const DEFAULT_RESULTS_FIELD = 'ml';
 export const FEATURE_IMPORTANCE = 'feature_importance';
 export const FEATURE_INFLUENCE = 'feature_influence';
 export const TOP_CLASSES = 'top_classes';
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
index 13826284e5242..a56345017258e 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts
@@ -30,11 +30,8 @@ import {
   getPredictionFieldName,
   getDefaultPredictionFieldName,
 } from '../../../../../../../common/util/analytics_utils';
-import {
-  DEFAULT_RESULTS_FIELD,
-  FEATURE_IMPORTANCE,
-  TOP_CLASSES,
-} from '../../../../common/constants';
+import { FEATURE_IMPORTANCE, TOP_CLASSES } from '../../../../common/constants';
+import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics';
 import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields';
 import { isRegressionAnalysis } from '../../../../common/analytics';
 import { extractErrorMessage } from '../../../../../../../common/util/errors';
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts
index 24649ae5f1e71..151e5ea4e6feb 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts
@@ -29,7 +29,8 @@ import { SavedSearchQuery } from '../../../../../contexts/ml';
 import { getToastNotifications } from '../../../../../util/dependency_cache';
 
 import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common';
-import { DEFAULT_RESULTS_FIELD, FEATURE_INFLUENCE } from '../../../../common/constants';
+import { FEATURE_INFLUENCE } from '../../../../common/constants';
+import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics';
 import { sortExplorationResultsFields, ML__ID_COPY } from '../../../../common/fields';
 
 import { getFeatureCount, getOutlierScoreFieldName } from './common';
diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx
index 60c699ba0d370..ce24892c9de45 100644
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx
@@ -12,7 +12,7 @@ import { IIndexPattern } from 'src/plugins/data/common';
 import { DeepReadonly } from '../../../../../../../common/types/common';
 import { DataFrameAnalyticsConfig, isOutlierAnalysis } from '../../../../common';
 import { isClassificationAnalysis, isRegressionAnalysis } from '../../../../common/analytics';
-import { DEFAULT_RESULTS_FIELD } from '../../../../common/constants';
+import { DEFAULT_RESULTS_FIELD } from '../../../../../../../common/constants/data_frame_analytics';
 import { useMlKibana, useNavigateToPath } from '../../../../../contexts/kibana';
 import { DEFAULT_NUM_TOP_FEATURE_IMPORTANCE_VALUES } from '../../hooks/use_create_analytics_form';
 import { State } from '../../hooks/use_create_analytics_form/state';
diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
index 9c6a47d53ac05..94f54a5654873 100644
--- a/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
+++ b/x-pack/plugins/ml/server/models/data_frame_analytics/feature_importance.ts
@@ -5,8 +5,12 @@
  */
 
 import { IScopedClusterClient } from 'kibana/server';
-import { getPredictionFieldName, isRegressionAnalysis } from '../../../common/util/analytics_utils';
-
+import {
+  getDefaultPredictionFieldName,
+  getPredictionFieldName,
+  isRegressionAnalysis,
+} from '../../../common/util/analytics_utils';
+import { DEFAULT_RESULTS_FIELD } from '../../../common/constants/data_frame_analytics';
 // Obtains data for the data frame analytics feature importance functionalities
 // such as baseline, decision paths, or importance summary.
 export function analyticsFeatureImportanceProvider({
@@ -20,7 +24,13 @@ export function analyticsFeatureImportanceProvider({
     const jobConfig = body.data_frame_analytics[0];
     if (!isRegressionAnalysis) return undefined;
     const destinationIndex = jobConfig.dest.index;
-    const predictionField = getPredictionFieldName(jobConfig.analysis);
+    const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
+    const mlResultsField = jobConfig.dest?.results_field ?? DEFAULT_RESULTS_FIELD;
+    const predictedField = `${mlResultsField}.${
+      predictionFieldName ? predictionFieldName : getDefaultPredictionFieldName(jobConfig.analysis)
+    }`;
+    const isTrainingField = `${mlResultsField}.is_training`;
+
     const params = {
       index: destinationIndex,
       size: 0,
@@ -30,7 +40,7 @@ export function analyticsFeatureImportanceProvider({
             filter: [
               {
                 term: {
-                  'ml.is_training': true,
+                  [isTrainingField]: true,
                 },
               },
             ],
@@ -39,7 +49,7 @@ export function analyticsFeatureImportanceProvider({
         aggs: {
           featureImportanceBaseline: {
             avg: {
-              field: `ml.${predictionField}`,
+              field: predictedField,
             },
           },
         },