[ES|QL] Dashboard variables (#202875)

## Summary

Closes #203967

Supports dashboard variables in ES|QL charts.

This PR introduces the first phase of ES|QL controls. In this phase:
- the flow starts from Lens ES|QL editor (and no vice-versa, this will
happen on a later phase after we discuss some technical details with ES)
- it is only available for dashboards (we want to include them in other
apps as Discover but this is the next phase driven by the presentation
team)
- it supports variables for intervals, fields and values. I haven't
added support for functions. I am going to do it after this PR being
merged (there are some business questions I want to answer first)

For more info check this
[deck](https://docs.google.com/presentation/d/1qSbWLSoC5SseXuLix763vpp8sa7ikp3pQTbHImEHBoc)


![meow](https://github.com/user-attachments/assets/c101a257-fbe4-44e6-9686-18012f39e8c1)

### Implementation details

- There is a new service, the ESQLVariables service that is responsible
for ES|QL variables. I isolated this to a new plugin owned by the ES|QL
team for cleaner code and for avoiding circular dependencies
- A new ESQL_CONTROL type got created. It follows the exact same logic
as the rest controls. No changes in the architecture here.
- The creation of the controls (the control forms) have been added in
the esql plugin.
- Lens has small changes:
   -  The support of variables in the textBased datasource
- Two callbacks needed to be called after the creation / cancellation of
an ES|QL control


### Types of ES|QL variables 

We have 2 types:

- Static Values (the user gives a list of values with his own
responsibility). As the flow starts from the editor we can identify what
they most possibly want to do and we give the user some options but they
have the freedom to do as they want. A basic validation has been added
too.
- Values from an ES|QL query (the user gives an ES|QL query that
generates the values). As the flow starts from the editor we can suggest
a query for the users but they can always change it as they wish.

<img width="1168" alt="image"
src="https://github.com/user-attachments/assets/cc28beb8-111c-43ad-9f26-865bc62ae512"
/>

### Example of a control creation from the editor

![meow](https://github.com/user-attachments/assets/09fa0e21-98cd-4160-b271-4f8ed0a91bf7)


### Release note
ES|QL charts now allow the creation of controls in dashboards. You can
control a part of the query such as a field, an interval or a value.

### Checklist
- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Andrea Del Rio <[email protected]>
Co-authored-by: Devon Thomson <[email protected]>

.buildkite/ftr_platform_stateful_configs.yml

@@ -66,6 +66,7 @@ enabled:
   - test/functional/apps/dashboard/group4/config.ts
   - test/functional/apps/dashboard/group5/config.ts
   - test/functional/apps/dashboard/group6/config.ts
+  - test/functional/apps/dashboard/esql_controls/config.ts
   - test/functional/apps/discover/ccs_compatibility/config.ts
   - test/functional/apps/discover/embeddable/config.ts
   - test/functional/apps/discover/esql/config.ts

.github/CODEOWNERS

@@ -463,6 +463,7 @@ src/platform/packages/shared/kbn-es-types @elastic/kibana-core @elastic/obs-know
 src/platform/packages/shared/kbn-esql-ast @elastic/kibana-esql
 src/platform/packages/shared/kbn-esql-utils @elastic/kibana-esql
 src/platform/packages/shared/kbn-esql-validation-autocomplete @elastic/kibana-esql
+src/platform/packages/shared/kbn-esql-variables-types @elastic/kibana-esql
 src/platform/packages/shared/kbn-event-annotation-common @elastic/kibana-visualizations
 src/platform/packages/shared/kbn-event-annotation-components @elastic/kibana-visualizations
 src/platform/packages/shared/kbn-field-types @elastic/kibana-data-discovery

package.json

@@ -492,6 +492,7 @@
     "@kbn/esql-utils": "link:src/platform/packages/shared/kbn-esql-utils",
     "@kbn/esql-validation-autocomplete": "link:src/platform/packages/shared/kbn-esql-validation-autocomplete",
     "@kbn/esql-validation-example-plugin": "link:examples/esql_validation_example",
+    "@kbn/esql-variables-types": "link:src/platform/packages/shared/kbn-esql-variables-types",
     "@kbn/eui-provider-dev-warning": "link:test/plugin_functional/plugins/eui_provider_dev_warning",
     "@kbn/event-annotation-common": "link:src/platform/packages/shared/kbn-event-annotation-common",
     "@kbn/event-annotation-components": "link:src/platform/packages/shared/kbn-event-annotation-components",

src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx

@@ -19,6 +19,7 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import moment from 'moment';
+import { isEqual } from 'lodash';
 import { CodeEditor, CodeEditorProps } from '@kbn/code-editor';
 import type { CoreStart } from '@kbn/core/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
@@ -30,8 +31,9 @@ import memoize from 'lodash/memoize';
 import React, { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { createPortal } from 'react-dom';
 import { css } from '@emotion/react';
-import { ESQLRealField } from '@kbn/esql-validation-autocomplete';
+import { ESQLRealField, ESQLControlVariable } from '@kbn/esql-validation-autocomplete';
 import { FieldType } from '@kbn/esql-validation-autocomplete/src/definitions/types';
+import { ESQLVariableType } from '@kbn/esql-validation-autocomplete';
 import { EditorFooter } from './editor_footer';
 import { fetchFieldsFromESQL } from './fetch_fields_from_esql';
 import {
@@ -59,6 +61,25 @@ import './overwrite.scss';
 // for editor width smaller than this value we want to start hiding some text
 const BREAKPOINT_WIDTH = 540;
 
+const triggerControl = async (
+  queryString: string,
+  variableType: ESQLVariableType,
+  position: monaco.Position | null | undefined,
+  uiActions: ESQLEditorDeps['uiActions'],
+  esqlVariables?: ESQLControlVariable[],
+  onSaveControl?: ESQLEditorProps['onSaveControl'],
+  onCancelControl?: ESQLEditorProps['onCancelControl']
+) => {
+  await uiActions.getTrigger('ESQL_CONTROL_TRIGGER').exec({
+    queryString,
+    variableType,
+    cursorPosition: position,
+    esqlVariables,
+    onSaveControl,
+    onCancelControl,
+  });
+};
+
 export const ESQLEditor = memo(function ESQLEditor({
   query,
   onTextLangQueryChange,
@@ -78,6 +99,10 @@ export const ESQLEditor = memo(function ESQLEditor({
   hasOutline,
   displayDocumentationAsFlyout,
   disableAutoFocus,
+  onSaveControl,
+  onCancelControl,
+  supportsControls,
+  esqlVariables,
 }: ESQLEditorProps) {
   const popoverRef = useRef<HTMLDivElement>(null);
   const datePickerOpenStatusRef = useRef<boolean>(false);
@@ -91,8 +116,10 @@ export const ESQLEditor = memo(function ESQLEditor({
     core,
     fieldsMetadata,
     uiSettings,
+    uiActions,
   } = kibana.services;
 
+  const variablesService = kibana.services?.esql?.variablesService;
   const histogramBarTarget = uiSettings?.get('histogram:barTarget') ?? 50;
   const [code, setCode] = useState<string>(query.esql ?? '');
   // To make server side errors less "sticky", register the state of the code when submitting
@@ -187,6 +214,23 @@ export const ESQLEditor = memo(function ESQLEditor({
     }
   }, [code, query.esql]);
 
+  // Enable the variables service if the feature is supported in the consumer app
+  useEffect(() => {
+    if (supportsControls) {
+      variablesService?.enableSuggestions();
+
+      const variables = variablesService?.esqlVariables;
+      if (!isEqual(variables, esqlVariables)) {
+        variablesService?.clearVariables();
+        esqlVariables?.forEach((variable) => {
+          variablesService?.addVariable(variable);
+        });
+      }
+    } else {
+      variablesService?.disableSuggestions();
+    }
+  }, [variablesService, supportsControls, esqlVariables]);
+
   const toggleHistory = useCallback((status: boolean) => {
     setIsHistoryOpen(status);
   }, []);
@@ -230,6 +274,45 @@ export const ESQLEditor = memo(function ESQLEditor({
     openTimePickerPopover();
   });
 
+  monaco.editor.registerCommand('esql.control.time_literal.create', async (...args) => {
+    const position = editor1.current?.getPosition();
+    await triggerControl(
+      query.esql,
+      ESQLVariableType.TIME_LITERAL,
+      position,
+      uiActions,
+      esqlVariables,
+      onSaveControl,
+      onCancelControl
+    );
+  });
+
+  monaco.editor.registerCommand('esql.control.fields.create', async (...args) => {
+    const position = editor1.current?.getPosition();
+    await triggerControl(
+      query.esql,
+      ESQLVariableType.FIELDS,
+      position,
+      uiActions,
+      esqlVariables,
+      onSaveControl,
+      onCancelControl
+    );
+  });
+
+  monaco.editor.registerCommand('esql.control.values.create', async (...args) => {
+    const position = editor1.current?.getPosition();
+    await triggerControl(
+      query.esql,
+      ESQLVariableType.VALUES,
+      position,
+      uiActions,
+      esqlVariables,
+      onSaveControl,
+      onCancelControl
+    );
+  });
+
   const styles = esqlEditorStyles(
     euiTheme,
     editorHeight,
@@ -373,13 +456,20 @@ export const ESQLEditor = memo(function ESQLEditor({
       },
       // @ts-expect-error To prevent circular type import, type defined here is partial of full client
       getFieldsMetadata: fieldsMetadata?.getClient(),
+      getVariablesByType: (type: ESQLVariableType) => {
+        return variablesService?.esqlVariables.filter((variable) => variable.type === type);
+      },
+      canSuggestVariables: () => {
+        return variablesService?.areSuggestionsEnabled ?? false;
+      },
       getJoinIndices: kibana.services?.esql?.getJoinIndicesAutocomplete,
     };
     return callbacks;
   }, [
+    fieldsMetadata,
+    dataSourcesCache,
     query.esql,
     memoizedSources,
-    dataSourcesCache,
     dataViews,
     core,
     esqlFieldsCache,
@@ -388,7 +478,7 @@ export const ESQLEditor = memo(function ESQLEditor({
     abortController,
     indexManagementApiService,
     histogramBarTarget,
-    fieldsMetadata,
+    variablesService,
     kibana.services?.esql?.getJoinIndicesAutocomplete,
   ]);
 

src/platform/packages/private/kbn-esql-editor/src/types.ts

@@ -15,6 +15,8 @@ import type { IndexManagementPluginSetup } from '@kbn/index-management-shared-ty
 import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
 import type { UsageCollectionStart } from '@kbn/usage-collection-plugin/public';
 import type { Storage } from '@kbn/kibana-utils-plugin/public';
+import type { UiActionsStart } from '@kbn/ui-actions-plugin/public';
+import type { ESQLControlVariable } from '@kbn/esql-validation-autocomplete';
 
 export interface ESQLEditorProps {
   /** The aggregate type query */
@@ -69,6 +71,16 @@ export interface ESQLEditorProps {
 
   /** The component by default focuses on the editor when it is mounted, this flag disables it**/
   disableAutoFocus?: boolean;
+  /** The editor supports the creation of controls,
+   * This flag should be set to true to display the "Create control" suggestion
+   **/
+  supportsControls?: boolean;
+  /** Function to be called after the control creation **/
+  onSaveControl?: (controlState: Record<string, unknown>, updatedQuery: string) => Promise<void>;
+  /** Function to be called after cancelling the control creation **/
+  onCancelControl?: () => void;
+  /** The available ESQL variables from the page context this editor was opened in */
+  esqlVariables?: ESQLControlVariable[];
 }
 
 export interface JoinIndicesAutocompleteResult {
@@ -81,15 +93,26 @@ export interface JoinIndexAutocompleteItem {
   aliases: string[];
 }
 
+interface ESQLVariableService {
+  areSuggestionsEnabled: boolean;
+  esqlVariables: ESQLControlVariable[];
+  enableSuggestions: () => void;
+  disableSuggestions: () => void;
+  clearVariables: () => void;
+  addVariable: (variable: ESQLControlVariable) => void;
+}
+
 export interface EsqlPluginStartBase {
   getJoinIndicesAutocomplete: () => Promise<JoinIndicesAutocompleteResult>;
+  variablesService: ESQLVariableService;
 }
 
 export interface ESQLEditorDeps {
   core: CoreStart;
   dataViews: DataViewsPublicPluginStart;
   expressions: ExpressionsStart;
   storage: Storage;
+  uiActions: UiActionsStart;
   indexManagementApiService?: IndexManagementPluginSetup['apiService'];
   fieldsMetadata?: FieldsMetadataPublicStart;
   usageCollection?: UsageCollectionStart;

src/platform/packages/private/kbn-esql-editor/tsconfig.json

@@ -32,6 +32,7 @@
     "@kbn/usage-collection-plugin",
     "@kbn/content-management-favorites-common",
     "@kbn/kibana-utils-plugin",
+    "@kbn/ui-actions-plugin",
     "@kbn/shared-ux-table-persist"
   ],
   "exclude": [

src/platform/packages/shared/kbn-es-query/src/expressions/types.ts

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-
+import type { ESQLControlVariable } from '@kbn/esql-validation-autocomplete';
 import { Filter, Query, TimeRange } from '../filters';
 
 export interface ExecutionContextSearch {
@@ -15,4 +15,5 @@ export interface ExecutionContextSearch {
   query?: Query | Query[];
   timeRange?: TimeRange;
   disableWarningToasts?: boolean;
+  esqlVariables?: ESQLControlVariable[];
 }

src/platform/packages/shared/kbn-es-query/tsconfig.json

@@ -15,7 +15,8 @@
   "kbn_references": [
     "@kbn/utility-types",
     "@kbn/i18n",
-    "@kbn/safer-lodash-set"
+    "@kbn/safer-lodash-set",
+    "@kbn/esql-validation-autocomplete"
   ],
   "exclude": [
     "target/**/*",

src/platform/packages/shared/kbn-es-types/src/search.ts

@@ -691,5 +691,7 @@ export interface ESQLSearchParams {
   locale?: string;
   include_ccs_metadata?: boolean;
   dropNullColumns?: boolean;
-  params?: estypes.ScalarValue[] | Array<Record<string, string | undefined>>;
+  params?:
+    | estypes.ScalarValue[]
+    | Array<Record<string, string | number | Record<string, string | number> | undefined>>;
 }

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -25,6 +25,7 @@ export {
   getTimeFieldFromESQLQuery,
   getStartEndParams,
   hasStartEndParams,
+  getNamedParams,
   prettifyQuery,
   isQueryWrappedByPipes,
   retrieveMetadataColumns,
@@ -35,6 +36,7 @@ export {
   TextBasedLanguages,
   sanitazeESQLInput,
   queryCannotBeSampled,
+  mapVariableToColumn,
 } from './src';
 
 export { ENABLE_ESQL, FEEDBACK_LINK } from './constants';

src/platform/packages/shared/kbn-esql-utils/src/index.ts

@@ -21,6 +21,7 @@ export {
   isQueryWrappedByPipes,
   retrieveMetadataColumns,
   getQueryColumnsFromESQLQuery,
+  mapVariableToColumn,
 } from './utils/query_parsing_helpers';
 export { queryCannotBeSampled } from './utils/query_cannot_be_sampled';
 export { appendToESQLQuery, appendWhereClauseToESQLQuery } from './utils/append_to_query';
@@ -31,6 +32,7 @@ export {
   formatESQLColumns,
   getStartEndParams,
   hasStartEndParams,
+  getNamedParams,
 } from './utils/run_query';
 export {
   isESQLColumnSortable,